services/prometheus: add blackbox exporter for dns_soa and http check

nixos/modules/networking/ports.nix

@@ -32,6 +32,7 @@ in
       caddy-admin = 4050;
       prometheus = 4060;
       prometheus-node-exporter = 4070;
+      prometheus-blackbox-exporter = 4071;
 
       # public ports
       enthalpy-wireguard-reimu-aston = 13101;

nixos/profiles/services/prometheus/server.nix

@@ -3,6 +3,7 @@
 {
   config,
   lib,
+  pkgs,
   data,
   ...
 }:
@@ -10,6 +11,24 @@ let
   common = import ../../../../zones/common.nix;
   publicHosts = lib.filterAttrs (_name: value: value.endpoints != [ ]) common.hosts;
   targets = lib.mapAttrsToList (name: _value: "${name}.rebmit.link") publicHosts;
+  primaryNameserver = "${common.primary}.rebmit.link";
+  nameservers = map (ns: "${ns}.rebmit.link") common.nameservers;
+  relabel_configs = [
+    {
+      source_labels = [ "__address__" ];
+      target_label = "__param_target";
+    }
+    {
+      source_labels = [ "__param_target" ];
+      target_label = "instance";
+    }
+    {
+      target_label = "__address__";
+      replacement =
+        with config.services.prometheus.exporters.blackbox;
+        "${listenAddress}:${toString port}";
+    }
+  ];
 in
 {
   sops.secrets."prometheus/password" = {
@@ -39,6 +58,35 @@ in
         };
         static_configs = [ { inherit targets; } ];
       }
+      {
+        job_name = "dns";
+        scheme = "http";
+        metrics_path = "/probe";
+        params = {
+          module = [ "dns_soa" ];
+        };
+        static_configs = [ { targets = nameservers; } ];
+        inherit relabel_configs;
+      }
+      {
+        job_name = "http";
+        scheme = "http";
+        metrics_path = "/probe";
+        params = {
+          module = [ "http_2xx" ];
+        };
+        static_configs = [
+          {
+            targets = [
+              "https://rebmit.moe"
+              "https://keycloak.rebmit.moe"
+              "https://matrix.rebmit.moe"
+              "https://miniflux.rebmit.moe"
+            ];
+          }
+        ];
+        inherit relabel_configs;
+      }
     ];
     rules = lib.singleton (
       builtins.toJSON {
@@ -63,6 +111,11 @@ in
                 alert = "UnitFailed";
                 expr = ''node_systemd_unit_state{state="failed"} == 1'';
               }
+              {
+                alert = "ZoneStale";
+                expr = ''probe_dns_serial{instance="${primaryNameserver}"} != ignoring(instance) group_right() probe_dns_serial'';
+                for = "5m";
+              }
             ];
           }
         ];
@@ -70,6 +123,26 @@ in
     );
   };
 
+  services.prometheus.exporters.blackbox = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = config.networking.ports.prometheus-blackbox-exporter;
+    configFile = (pkgs.formats.yaml { }).generate "config.yml" {
+      modules = {
+        http_2xx = {
+          prober = "http";
+        };
+        dns_soa = {
+          prober = "dns";
+          dns = {
+            query_name = "rebmit.moe";
+            query_type = "SOA";
+          };
+        };
+      };
+    };
+  };
+
   sops.secrets."cloudflare_origin_prometheus_private_key" = {
     opentofu = {
       enable = true;