From b4ea4daa0699d4e95eb6f70be740203d7f4d6058 Mon Sep 17 00:00:00 2001
From: Azat Bahawi <azat@bahawi.net>
Date: Thu, 16 Mar 2023 03:35:21 +0300
Subject: 2023-03-16

---
 modules/nixos/monitoring/default.nix | 34 +++++++++++++++++++++++++++++
 modules/nixos/promtail.nix           | 42 +++++++++++++++++++++++++-----------
 2 files changed, 64 insertions(+), 12 deletions(-)

(limited to 'modules')

diff --git a/modules/nixos/monitoring/default.nix b/modules/nixos/monitoring/default.nix
index a48701e..1108be6 100644
--- a/modules/nixos/monitoring/default.nix
+++ b/modules/nixos/monitoring/default.nix
@@ -70,6 +70,40 @@ in {
             options.path = ./dashboards/postgresql.json;
           }
         ];
+
+        alerting = {
+          contactPoints.settings.contactPoints = [
+            {
+              name = "Alertmanager";
+              receivers = [
+                {
+                  uid = 1;
+                  type = "prometheus-alertmanager";
+                  settings.url = "https://${config.nixfiles.modules.alertmanager.domain}";
+                }
+              ];
+            }
+          ];
+          muteTimings.settings.muteTimes = [
+            {
+              name = "Sleep";
+              time_intervals = [
+                {
+                  times = [
+                    {
+                      start_time = "23:00";
+                      end_time = "24:00";
+                    }
+                    {
+                      start_time = "00:00";
+                      end_time = "09:00";
+                    }
+                  ];
+                }
+              ];
+            }
+          ];
+        };
       };
 
       loki.configuration.ruler.alertmanager_url = "https://${config.nixfiles.modules.alertmanager.domain}";
diff --git a/modules/nixos/promtail.nix b/modules/nixos/promtail.nix
index c038d7b..a3a6fe9 100644
--- a/modules/nixos/promtail.nix
+++ b/modules/nixos/promtail.nix
@@ -37,6 +37,8 @@ in {
         clients = [
           {
             url = "${cfg.loki.url}/loki/api/v1/push";
+            batchwait = "15s";
+            batchsize = 1000000;
             external_labels.host_id = config.networking.hostId;
           }
         ];
@@ -46,7 +48,7 @@ in {
         scrape_configs = [
           {
             job_name = "journal";
-            journal.max_age = "24h";
+            journal.max_age = "12h";
             relabel_configs =
               map (n: let
                 label = toLower n;
@@ -63,18 +65,18 @@ in {
                 # [2]: https://github.com/coreos/go-systemd/blob/main/sdjournal/journal.go#L335
 
                 "MESSAGE"
-                "MESSAGE_ID"
-                "PRIORITY"
+                # "MESSAGE_ID"
+                # "PRIORITY"
                 # "CODE_FILE"
                 # "CODE_LINE"
                 # "CODE_FUNC"
-                "ERRNO"
-                "SYSLOG_FACILITY"
+                # "ERRNO"
+                # "SYSLOG_FACILITY"
                 "SYSLOG_IDENTIFIER"
-                "SYSLOG_PID"
-                "_PID"
-                "_UID"
-                "_GID"
+                # "SYSLOG_PID"
+                # "_PID"
+                # "_UID"
+                # "_GID"
                 # "_COMM"
                 # "_EXE"
                 "_CMDLINE"
@@ -89,14 +91,30 @@ in {
                 # "_SYSTEMD_SLICE"
                 # "_SELINUX_CONTEXT"
                 # "_SOURCE_REALTIME_TIMESTAMP"
-                "_BOOT_ID"
-                "_MACHINE_ID"
+                # "_BOOT_ID"
+                # "_MACHINE_ID"
                 "_HOSTNAME"
-                "_TRANSPORT"
+                # "_TRANSPORT"
                 # "__CURSOR"
                 # "__REALTIME_TIMESTAMP"
                 # "__MONOTONIC_TIMESTAMP"
               ];
+            pipeline_stages = [
+              {
+                match = {
+                  selector = ''{systemd_unit="dendrite.service"} |~ ".*Failed to fetch key for server.*"'';
+                  action = "drop";
+                  drop_counter_reason = "noisy_error";
+                };
+              }
+              {
+                match = {
+                  selector = ''{systemd_unit="dendrite.service"} |~ ".*could not download key for.*"'';
+                  action = "drop";
+                  drop_counter_reason = "noisy_error";
+                };
+              }
+            ];
           }
         ];
       };
-- 
cgit v1.2.3