diff --git a/nixos/modules/networking/ports.nix b/nixos/modules/networking/ports.nix index 92cc620..449a5e9 100644 --- a/nixos/modules/networking/ports.nix +++ b/nixos/modules/networking/ports.nix @@ -32,6 +32,7 @@ in caddy-admin = 4050; prometheus = 4060; prometheus-node-exporter = 4070; + prometheus-blackbox-exporter = 4071; # public ports enthalpy-wireguard-reimu-aston = 13101; diff --git a/nixos/profiles/services/prometheus/server.nix b/nixos/profiles/services/prometheus/server.nix index bfc3228..0b9fc87 100644 --- a/nixos/profiles/services/prometheus/server.nix +++ b/nixos/profiles/services/prometheus/server.nix @@ -3,6 +3,7 @@ { config, lib, + pkgs, data, ... }: @@ -10,6 +11,24 @@ let common = import ../../../../zones/common.nix; publicHosts = lib.filterAttrs (_name: value: value.endpoints != [ ]) common.hosts; targets = lib.mapAttrsToList (name: _value: "${name}.rebmit.link") publicHosts; + primaryNameserver = "${common.primary}.rebmit.link"; + nameservers = map (ns: "${ns}.rebmit.link") common.nameservers; + relabel_configs = [ + { + source_labels = [ "__address__" ]; + target_label = "__param_target"; + } + { + source_labels = [ "__param_target" ]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = + with config.services.prometheus.exporters.blackbox; + "${listenAddress}:${toString port}"; + } + ]; in { sops.secrets."prometheus/password" = { @@ -39,6 +58,35 @@ in }; static_configs = [ { inherit targets; } ]; } + { + job_name = "dns"; + scheme = "http"; + metrics_path = "/probe"; + params = { + module = [ "dns_soa" ]; + }; + static_configs = [ { targets = nameservers; } ]; + inherit relabel_configs; + } + { + job_name = "http"; + scheme = "http"; + metrics_path = "/probe"; + params = { + module = [ "http_2xx" ]; + }; + static_configs = [ + { + targets = [ + "https://rebmit.moe" + "https://keycloak.rebmit.moe" + "https://matrix.rebmit.moe" + "https://miniflux.rebmit.moe" + ]; + } + ]; + inherit relabel_configs; + } ]; rules = lib.singleton ( builtins.toJSON { @@ -63,6 +111,11 @@ in alert = "UnitFailed"; expr = ''node_systemd_unit_state{state="failed"} == 1''; } + { + alert = "ZoneStale"; + expr = ''probe_dns_serial{instance="${primaryNameserver}"} != ignoring(instance) group_right() probe_dns_serial''; + for = "5m"; + } ]; } ]; @@ -70,6 +123,26 @@ in ); }; + services.prometheus.exporters.blackbox = { + enable = true; + listenAddress = "127.0.0.1"; + port = config.networking.ports.prometheus-blackbox-exporter; + configFile = (pkgs.formats.yaml { }).generate "config.yml" { + modules = { + http_2xx = { + prober = "http"; + }; + dns_soa = { + prober = "dns"; + dns = { + query_name = "rebmit.moe"; + query_type = "SOA"; + }; + }; + }; + }; + }; + sops.secrets."cloudflare_origin_prometheus_private_key" = { opentofu = { enable = true;