add monitoring

config/default.nix

@@ -11,6 +11,7 @@
     ./http
     ./search
     ./security
+    ./monitoring
   ];
   system.stateVersion = "25.05";
 }

config/monitoring/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./grafana.nix
+    ./prometheus.nix
+  ];
+}

config/monitoring/grafana.nix

@@ -0,0 +1,93 @@
+{config, ...}: let
+  cfg = config.services.grafana;
+  owner = config.customOps.owner;
+  domain = config.customOps.domain.fqdn;
+  hostname = config.networking.hostName;
+  subdomain = "${hostname}.${domain}";
+
+  dashboardsDir = ./dashboards;
+  dashboardFiles = builtins.attrNames (builtins.readDir dashboardsDir);
+in {
+  sops.secrets."grafana/admin_password".owner = "grafana";
+
+  services.nginx.virtualHosts."${subdomain}" = {
+    locations."/" = {
+      proxyPass = "http://${
+        toString cfg.settings.server.http_addr
+      }:${
+        toString cfg.settings.server.http_port
+      }";
+      proxyWebsockets = true;
+      recommendedProxySettings = true;
+    };
+    forceSSL = true;
+    enableACME = true;
+  };
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        http_addr = "127.0.0.1";
+        http_port = 3030;
+        domain = subdomain;
+        enforce_domain = true;
+        enable_gzip = true;
+      };
+      security = {
+        disable_initial_admin_creation = false;
+        admin_user = owner;
+        admin_email = "admin.grafana@${domain}";
+        admin_password = "$__file{${
+          config.sops.secrets."grafana/admin_password".path
+        }}";
+        disable_brute_force_login_protection = false;
+        brute_force_login_protection_max_attempts = 3;
+        disable_username_login_protection = false;
+        disable_ip_address_login_protection = false;
+        cookie_secure = true;
+        cookie_samesite = "strict";
+      };
+      analytics = {
+        reporting_enabled = false;
+        check_for_updates = false;
+      };
+    };
+
+    provision = {
+      enable = true;
+      dashboards.settings.providers = [
+        {
+          name = "dashboards";
+          disableDeletion = true;
+          options = {
+            path = "/etc/grafana-dashboards";
+            foldersFromFilesStructure = true;
+          };
+        }
+      ];
+
+      datasources.settings.datasources = [
+        {
+          name = "prometheus";
+          type = "prometheus";
+          url = "http://${
+            config.services.prometheus.listenAddress
+          }:${
+            toString config.services.prometheus.port
+          }";
+          isDefault = true;
+          editable = false;
+        }
+      ];
+    };
+  };
+
+  environment.etc = builtins.listToAttrs (map (
+      name: {
+        name = "grafana-dashboards/" + name;
+        value = {source = dashboardsDir + "/" + name;};
+      }
+    )
+    dashboardFiles);
+}

config/monitoring/prometheus.nix

@@ -0,0 +1,40 @@
+{config, ...}: {
+  services.prometheus = {
+    enable = true;
+    port = 9100;
+    globalConfig.scrape_interval = "1m";
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [
+          {
+            targets = [
+              "localhost:${
+                toString
+                config.services.prometheus.exporters.node.port
+              }"
+            ];
+          }
+        ];
+      }
+    ];
+
+    exporters = {
+      node = {
+        enable = true;
+        port = 9101;
+        enabledCollectors = [
+          "ethtool"
+          "softirqs"
+          "systemd"
+          "tcpstat"
+          "wifi"
+        ];
+        extraFlags = [
+          "--collector.ntp.protocol-version=4"
+          "--no-collector.mdadm"
+        ];
+      };
+    };
+  };
+}

config/network.nix

@@ -26,6 +26,7 @@ in {
       MaxAdvertisedBandwidth = "100 MB";
       BandWidthRate = "100 MB";
       RelayBandwidthRate = "100 MB";
+      RelayBandwidthBurst = "100 MB";
 
       CookieAuthentication = true;
       AvoidDiskWrites = 1;