diff options
| -rw-r--r-- | modules/nixos/monitoring/default.nix | 34 | ||||
| -rw-r--r-- | modules/nixos/promtail.nix | 42 |
2 files changed, 64 insertions, 12 deletions
diff --git a/modules/nixos/monitoring/default.nix b/modules/nixos/monitoring/default.nix index a48701e..1108be6 100644 --- a/modules/nixos/monitoring/default.nix +++ b/modules/nixos/monitoring/default.nix @@ -70,6 +70,40 @@ in { options.path = ./dashboards/postgresql.json; } ]; + + alerting = { + contactPoints.settings.contactPoints = [ + { + name = "Alertmanager"; + receivers = [ + { + uid = 1; + type = "prometheus-alertmanager"; + settings.url = "https://${config.nixfiles.modules.alertmanager.domain}"; + } + ]; + } + ]; + muteTimings.settings.muteTimes = [ + { + name = "Sleep"; + time_intervals = [ + { + times = [ + { + start_time = "23:00"; + end_time = "24:00"; + } + { + start_time = "00:00"; + end_time = "09:00"; + } + ]; + } + ]; + } + ]; + }; }; loki.configuration.ruler.alertmanager_url = "https://${config.nixfiles.modules.alertmanager.domain}"; diff --git a/modules/nixos/promtail.nix b/modules/nixos/promtail.nix index c038d7b..a3a6fe9 100644 --- a/modules/nixos/promtail.nix +++ b/modules/nixos/promtail.nix @@ -37,6 +37,8 @@ in { clients = [ { url = "${cfg.loki.url}/loki/api/v1/push"; + batchwait = "15s"; + batchsize = 1000000; external_labels.host_id = config.networking.hostId; } ]; @@ -46,7 +48,7 @@ in { scrape_configs = [ { job_name = "journal"; - journal.max_age = "24h"; + journal.max_age = "12h"; relabel_configs = map (n: let label = toLower n; @@ -63,18 +65,18 @@ in { # [2]: https://github.com/coreos/go-systemd/blob/main/sdjournal/journal.go#L335 "MESSAGE" - "MESSAGE_ID" - "PRIORITY" + # "MESSAGE_ID" + # "PRIORITY" # "CODE_FILE" # "CODE_LINE" # "CODE_FUNC" - "ERRNO" - "SYSLOG_FACILITY" + # "ERRNO" + # "SYSLOG_FACILITY" "SYSLOG_IDENTIFIER" - "SYSLOG_PID" - "_PID" - "_UID" - "_GID" + # "SYSLOG_PID" + # "_PID" + # "_UID" + # "_GID" # "_COMM" # "_EXE" "_CMDLINE" @@ -89,14 +91,30 @@ in { # "_SYSTEMD_SLICE" # "_SELINUX_CONTEXT" # "_SOURCE_REALTIME_TIMESTAMP" - "_BOOT_ID" - "_MACHINE_ID" + # "_BOOT_ID" + # "_MACHINE_ID" "_HOSTNAME" - "_TRANSPORT" + # "_TRANSPORT" # "__CURSOR" # "__REALTIME_TIMESTAMP" # "__MONOTONIC_TIMESTAMP" ]; + pipeline_stages = [ + { + match = { + selector = ''{systemd_unit="dendrite.service"} |~ ".*Failed to fetch key for server.*"''; + action = "drop"; + drop_counter_reason = "noisy_error"; + }; + } + { + match = { + selector = ''{systemd_unit="dendrite.service"} |~ ".*could not download key for.*"''; + action = "drop"; + drop_counter_reason = "noisy_error"; + }; + } + ]; } ]; }; |