{
  config,
  lib,
  pkgs,
  ...
}:
with lib; let
  cfg = config.nixfiles.modules.monitoring;
in {
  options.nixfiles.modules.monitoring.enable = mkEnableOption ''
    a glue to provision a monitoring stack
  '';

  config = mkIf cfg.enable {
    nixfiles.modules = {
      alertmanager.enable = true;
      grafana.enable = true;
      loki.enable = true;
      prometheus.enable = true;
    };

    services = {
      grafana = {
        declarativePlugins = with pkgs.grafanaPlugins; [
          redis-app
          redis-datasource
          redis-explorer-app
        ];

        provision = {
          enable = true;

          # https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources
          datasources.settings.datasources = with config.nixfiles.modules; [
            {
              name = "Alertmanager";
              type = "alertmanager";
              access = "proxy";
              jsonData.implementation = "prometheus";
              url = "https://${alertmanager.domain}";
            }
            {
              name = "Loki";
              type = "loki";
              access = "proxy";
              url = "https://${loki.domain}";
              isDefault = true;
            }
            {
              name = "Prometheus";
              type = "prometheus";
              access = "proxy";
              url = "https://${prometheus.domain}";
            }
            (mkIf config.nixfiles.modules.redis.enable {
              name = "Redis";
              type = "redis-datasource";
              access = "proxy";
              url = with config.services.redis.servers.default; "redis://${bind}:${toString port}";
              jsonData.client = "standalone";
            })
          ];
          datasources.settings.deleteDatasources = [
            {
              name = "PostgreSQL";
              orgId = 1;
            }
          ];

          # https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards
          dashboards.settings.providers = [
            {
              name = "node";
              options.path = ./dashboards/node.json;
            }
            {
              name = "ntfy";
              options.path = ./dashboards/ntfy.json;
            }
            {
              name = "endlessh";
              options.path = ./dashboards/endlessh.json;
            }
            {
              name = "unbound";
              options.path = ./dashboards/unbound.json;
            }
            {
              name = "nginx";
              options.path = ./dashboards/nginx.json;
            }
            {
              name = "postgersql";
              options.path = ./dashboards/postgresql.json;
            }
            {
              name = "redis";
              options.path = ./dashboards/redis.json;
            }
            {
              name = "redis-streaming";
              options.path = ./dashboards/redis-streaming.json;
            }
          ];

          # https://grafana.com/docs/grafana/latest/administration/provisioning/#alerting
          alerting.contactPoints.settings.contactPoints = [
            {
              name = "Alertmanager";
              receivers = [
                {
                  uid = 1;
                  type = "prometheus-alertmanager";
                  settings.url = "https://${config.nixfiles.modules.alertmanager.domain}";
                }
              ];
            }
          ];
        };
      };

      loki.configuration.ruler.alertmanager_url = "https://${config.nixfiles.modules.alertmanager.domain}";

      prometheus = {
        scrapeConfigs = with my.configurations;
          mapAttrsToList
          (
            name: value: {
              job_name = name;
              static_configs = [
                {
                  targets = with value;
                    map (host:
                      concatStringsSep ":" [
                        (
                          if isAttrs host
                          then host.hostname
                          else host
                        )
                        (toString port)
                      ])
                    hosts;
                }
              ];
              relabel_configs =
                [
                  {
                    source_labels = ["__address__"];
                    regex = "([^:]+):\\d+";
                    target_label = "instance";
                  }
                ]
                ++ optionals (hasAttr "relabel" value) value.relabel;
            }
          )
          {
            promtail = {
              hosts = [manwe varda yavanna];
              inherit (config.nixfiles.modules.promtail) port;
            };
            ntfy = {
              hosts = [manwe];
              inherit (config.nixfiles.modules.ntfy.prometheus) port;
            };
            soju = {
              hosts = ["127.0.0.1"];
              inherit (config.nixfiles.modules.soju.prometheus) port;
            };
            endlessh-go = {
              hosts = [manwe varda yavanna];
              inherit (config.services.endlessh-go.prometheus) port;
            };
            nginx = {
              hosts = [manwe yavanna];
              inherit (config.services.prometheus.exporters.nginx) port;
            };
            node = {
              hosts = [manwe varda yavanna];
              inherit (config.services.prometheus.exporters.node) port;
            };
            postgres = {
              hosts = [manwe];
              inherit (config.services.prometheus.exporters.postgres) port;
            };
            redis = {
              hosts = [manwe];
              inherit (config.services.prometheus.exporters.redis) port;
            };
            unbound = {
              hosts = [manwe];
              inherit (config.services.prometheus.exporters.unbound) port;
            };
            wireguard = {
              hosts = [manwe];
              inherit (config.services.prometheus.exporters.wireguard) port;
            };
          };

        ruleFiles = [
          ./rules/nginx.yaml
          ./rules/node.yaml
          ./rules/postgres.yaml
          ./rules/redis.yaml
        ];

        alertmanagers = [
          {
            scheme = "https";
            static_configs = [
              {targets = [config.nixfiles.modules.alertmanager.domain];}
            ];
          }
        ];
      };
    };
  };
}