From a4a8a2adad019dadec04f6a379b5a19f3ff5bcb7 Mon Sep 17 00:00:00 2001 From: ibizaman Date: Sun, 26 Nov 2023 09:30:45 -0800 Subject: [PATCH] add test for monitoring that checks provisioning goes well --- flake.nix | 10 +++- modules/blocks/monitoring.nix | 88 ++++++++++++++++++++--------------- test/vm/monitoring.nix | 42 +++++++++++++++++ 3 files changed, 101 insertions(+), 39 deletions(-) create mode 100644 test/vm/monitoring.nix diff --git a/flake.nix b/flake.nix index 8bb849a..895028d 100644 --- a/flake.nix +++ b/flake.nix @@ -49,6 +49,13 @@ flattenAttrs = root: attrset: pkgs.lib.attrsets.foldlAttrs (acc: name: value: acc // { "${root}_${name}" = value; }) {} attrset; + + vm_test = name: path: flattenAttrs "vm_${name}" ( + import path { + inherit pkgs; + inherit (pkgs) lib; + } + ); in (rec { all = mergeTests [ modules @@ -64,7 +71,8 @@ ]); }; } - // (flattenAttrs "vm_postgresql" (import ./test/vm/postgresql.nix {inherit pkgs; inherit (pkgs) lib;})) + // (vm_test "postgresql" ./test/vm/postgresql.nix) + // (vm_test "monitoring" ./test/vm/monitoring.nix) ); } ); diff --git a/modules/blocks/monitoring.nix b/modules/blocks/monitoring.nix index 38c756f..abba7e4 100644 --- a/modules/blocks/monitoring.nix +++ b/modules/blocks/monitoring.nix @@ -9,12 +9,6 @@ in options.shb.monitoring = { enable = lib.mkEnableOption "selfhostblocks.monitoring"; - # sopsFile = lib.mkOption { - # type = lib.types.path; - # description = "Sops file location"; - # example = "secrets/monitoring.yaml"; - # }; - subdomain = lib.mkOption { type = lib.types.str; description = "Subdomain under which home-assistant will be served."; @@ -27,6 +21,24 @@ in example = "mydomain.com"; }; + grafanaPort = lib.mkOption { + type = lib.types.port; + description = "Port where Grafana listens to HTTP requests."; + default = 3000; + }; + + prometheusPort = lib.mkOption { + type = lib.types.port; + description = "Port where Prometheus listens to HTTP requests."; + default = 3001; + }; + + lokiPort = lib.mkOption { + type = lib.types.port; + description = "Port where Loki listens to HTTP requests."; + default = 3002; + }; + debugLog = lib.mkOption { type = lib.types.bool; description = "Set to true to enable debug logging of the infrastructure serving Grafana."; @@ -49,10 +61,22 @@ in contactPoints = lib.mkOption { type = lib.types.listOf lib.types.str; description = "List of email addresses to send alerts to"; + default = []; + }; + + adminPasswordFile = lib.mkOption { + type = lib.types.path; + description = "File containing the initial admin password."; + }; + + secretKeyFile = lib.mkOption { + type = lib.types.path; + description = "File containing the secret key used for signing."; }; smtp = lib.mkOption { - type = lib.types.submodule { + default = null; + type = lib.types.nullOr (lib.types.submodule { options = { from_address = lib.mkOption { type = lib.types.str; @@ -82,14 +106,14 @@ in description = "File containing the password to connect to the SMTP host."; }; }; - }; + }); }; }; config = lib.mkIf cfg.enable { assertions = [ { - assertion = builtins.length cfg.contactPoints > 0; + assertion = (!(isNull cfg.smtp)) -> builtins.length cfg.contactPoints > 0; message = "Must have at least one contact point for alerting"; } ]; @@ -115,15 +139,21 @@ in # password = "$__file{/run/secrets/homeassistant/dbpass}"; }; + security = { + secret_key = "$__file{${cfg.secretKeyFile}}"; + disable_initial_admin_creation = false; # Enable when LDAP support is configured. + admin_password = "$__file{${cfg.adminPasswordFile}}"; # Remove when LDAP support is configured. + }; + server = { http_addr = "127.0.0.1"; - http_port = 3000; + http_port = cfg.grafanaPort; domain = fqdn; root_url = "https://${fqdn}"; router_logging = cfg.debugLog; }; - smtp = { + smtp = lib.mkIf (!(isNull cfg.smtp)) { enabled = true; inherit (cfg.smtp) from_address from_name; host = "${cfg.smtp.host}:${toString cfg.smtp.port}"; @@ -175,33 +205,24 @@ in } ]; }; - alerting.contactPoints.settings = lib.mkIf ((builtins.length cfg.contactPoints) > 0) { + alerting.contactPoints.settings = { apiVersion = 1; contactPoints = [{ inherit (cfg) orgId; - name = "selfhostblocks-sysadmin"; - receivers = [{ + name = "grafana-default-email"; + receivers = lib.optionals ((builtins.length cfg.contactPoints) > 0) [{ uid = "sysadmin"; type = "email"; settings.addresses = lib.concatStringsSep ";" cfg.contactPoints; }]; }]; - deleteContactPoints = [ - { - inherit (cfg) orgId; - uid = "grafana-default-email"; - } - ]; }; alerting.policies.settings = { apiVersion = 1; policies = [{ inherit (cfg) orgId; - receiver = "selfhostblocks-sysadmin"; + receiver = "grafana-default-email"; group_by = [ "grafana_folder" "alertname" ]; - object_matchers = [ - [ "role" "=" "sysadmin" ] - ]; group_wait = "30s"; group_interval = "5m"; repeat_interval = "4h"; @@ -230,7 +251,7 @@ in services.prometheus = { enable = true; - port = 3001; + port = cfg.prometheusPort; }; services.loki = { @@ -239,7 +260,7 @@ in configuration = { auth_enabled = false; - server.http_listen_port = 3002; + server.http_listen_port = cfg.lokiPort; ingester = { lifecycler = { @@ -340,9 +361,9 @@ in enable = true; virtualHosts.${fqdn} = { - forceSSL = true; - sslCertificate = "/var/lib/acme/${cfg.domain}/cert.pem"; - sslCertificateKey = "/var/lib/acme/${cfg.domain}/key.pem"; + forceSSL = lib.mkIf config.shb.ssl.enable true; + sslCertificate = lib.mkIf config.shb.ssl.enable "/var/lib/acme/${cfg.domain}/cert.pem"; + sslCertificateKey = lib.mkIf config.shb.ssl.enable "/var/lib/acme/${cfg.domain}/key.pem"; locations."/" = { proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}"; proxyWebsockets = true; @@ -439,14 +460,5 @@ in listenAddress = "127.0.0.1"; }; services.nginx.statusPage = lib.mkDefault config.services.nginx.enable; - - # sops.secrets."grafana" = { - # inherit (cfg) sopsFile; - # mode = "0440"; - # owner = "grafana"; - # group = "grafana"; - # # path = "${config.services.home-assistant.configDir}/secrets.yaml"; - # restartUnits = [ "grafana.service" ]; - # }; }; } diff --git a/test/vm/monitoring.nix b/test/vm/monitoring.nix new file mode 100644 index 0000000..12723ce --- /dev/null +++ b/test/vm/monitoring.nix @@ -0,0 +1,42 @@ +{ pkgs, lib, ... }: +{ + # This test, although simple, makes sure all provisioning went fine. + auth = pkgs.nixosTest { + name = "monitoring-basic"; + + nodes.machine = { config, pkgs, ... }: { + imports = [ + { + options = { + shb.ssl.enable = lib.mkEnableOption "ssl"; + }; + } + ../../modules/blocks/postgresql.nix + ../../modules/blocks/monitoring.nix + ]; + + shb.monitoring = { + enable = true; + subdomain = "grafana"; + domain = "example.com"; + grafanaPort = 3000; + adminPasswordFile = pkgs.writeText "admin_password" "securepw"; + secretKeyFile = pkgs.writeText "secret_key" "secret_key"; + }; + }; + + testScript = { nodes, ... }: '' + start_all() + machine.wait_for_unit("grafana.service") + + def curl_req(password, wantStatus, endpoint): + response = machine.wait_until_succeeds("curl -i http://admin:{password}@localhost:3000{endpoint}".format(password=password, endpoint=endpoint), timeout=10) + if not response.startswith("HTTP/1.1 {wantStatus}".format(wantStatus=wantStatus)): + raise Exception("Wrong status, expected {}, got {}".format(wantStatus, response[9:12])) + return response + + curl_req("securepw", 200, "/api/org") + curl_req("wrong", 401, "/api/org") + ''; + }; +}