selfhostblocks/modules/blocks/monitoring.nix

{ config, pkgs, lib, ... }:

let
  cfg = config.shb.monitoring;

  contracts = pkgs.callPackage ../contracts {};

  fqdn = "${cfg.subdomain}.${cfg.domain}";
in
{
  options.shb.monitoring = {
    enable = lib.mkEnableOption "selfhostblocks.monitoring";

    subdomain = lib.mkOption {
      type = lib.types.str;
      description = "Subdomain under which home-assistant will be served.";
      example = "grafana";
    };

    domain = lib.mkOption {
      type = lib.types.str;
      description = "domain under which home-assistant will be served.";
      example = "mydomain.com";
    };

    ssl = lib.mkOption {
      description = "Path to SSL files";
      type = lib.types.nullOr contracts.ssl.certs;
      default = null;
    };

    grafanaPort = lib.mkOption {
      type = lib.types.port;
      description = "Port where Grafana listens to HTTP requests.";
      default = 3000;
    };

    prometheusPort = lib.mkOption {
      type = lib.types.port;
      description = "Port where Prometheus listens to HTTP requests.";
      default = 3001;
    };

    lokiPort = lib.mkOption {
      type = lib.types.port;
      description = "Port where Loki listens to HTTP requests.";
      default = 3002;
    };

    lokiMajorVersion = lib.mkOption {
      type = lib.types.enum [ 2 3 ];
      description = ''
        Switching from version 2 to 3 requires manual intervention
        https://grafana.com/docs/loki/latest/setup/upgrade/#main--unreleased. So this let's the user
        upgrade at their own pace.
      '';
      default = 2;
    };

    debugLog = lib.mkOption {
      type = lib.types.bool;
      description = "Set to true to enable debug logging of the infrastructure serving Grafana.";
      default = false;
      example = true;
    };

    orgId = lib.mkOption {
      type = lib.types.int;
      description = "Org ID where all self host blocks related config will be stored.";
      default = 1;
    };

    provisionDashboards = lib.mkOption {
      type = lib.types.bool;
      description = "Provision Self Host Blocks dashboards under 'Self Host Blocks' folder.";
      default = true;
    };

    contactPoints = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      description = "List of email addresses to send alerts to";
      default = [];
    };

    adminPasswordFile = lib.mkOption {
      type = lib.types.path;
      description = "File containing the initial admin password.";
    };

    secretKeyFile = lib.mkOption {
      type = lib.types.path;
      description = "File containing the secret key used for signing.";
    };

    smtp = lib.mkOption {
      description = "SMTP options.";
      default = null;
      type = lib.types.nullOr (lib.types.submodule {
        options = {
          from_address = lib.mkOption {
            type = lib.types.str;
            description = "SMTP address from which the emails originate.";
            example = "vaultwarden@mydomain.com";
          };
          from_name = lib.mkOption {
            type = lib.types.str;
            description = "SMTP name from which the emails originate.";
            default = "Vaultwarden";
          };
          host = lib.mkOption {
            type = lib.types.str;
            description = "SMTP host to send the emails to.";
          };
          port = lib.mkOption {
            type = lib.types.port;
            description = "SMTP port to send the emails to.";
            default = 25;
          };
          username = lib.mkOption {
            type = lib.types.str;
            description = "Username to connect to the SMTP host.";
          };
          passwordFile = lib.mkOption {
            type = lib.types.str;
            description = "File containing the password to connect to the SMTP host.";
          };
        };
      });
    };
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = (!(isNull cfg.smtp)) -> builtins.length cfg.contactPoints > 0;
        message = "Must have at least one contact point for alerting";
      }
    ];

    shb.postgresql.ensures = [
      {
        username = "grafana";
        database = "grafana";
      }
    ];

    services.grafana = {
      enable = true;

      settings = {
        database = {
          host = "/run/postgresql";
          user = "grafana";
          name = "grafana";
          type = "postgres";
          # Uses peer auth for local users, so we don't need a password.
          # Here's the syntax anyway for future refence:
          # password = "$__file{/run/secrets/homeassistant/dbpass}";
        };

        security = {
          secret_key = "$__file{${cfg.secretKeyFile}}";
          disable_initial_admin_creation = false; # Enable when LDAP support is configured.
          admin_password = "$__file{${cfg.adminPasswordFile}}"; # Remove when LDAP support is configured.
        };

        server = {
          http_addr = "127.0.0.1";
          http_port = cfg.grafanaPort;
          domain = fqdn;
          root_url = "https://${fqdn}";
          router_logging = cfg.debugLog;
        };

        smtp = lib.mkIf (!(isNull cfg.smtp)) {
          enabled = true;
          inherit (cfg.smtp) from_address from_name;
          host = "${cfg.smtp.host}:${toString cfg.smtp.port}";
          user = cfg.smtp.username;
          password = "$__file{${cfg.smtp.passwordFile}}";
        };
      };
    };

    services.grafana.provision = {
      dashboards.settings = lib.mkIf cfg.provisionDashboards {
        apiVersion = 1;
        providers = [{
          folder = "Self Host Blocks";
          options.path = ./monitoring/dashboards;
          allowUiUpdates = true;
          disableDeletion = true;
        }];
      };
      datasources.settings = {
        apiVersion = 1;
        datasources = [
          {
            inherit (cfg) orgId;
            name = "Prometheus";
            type = "prometheus";
            url = "http://127.0.0.1:${toString config.services.prometheus.port}";
            uid = "df80f9f5-97d7-4112-91d8-72f523a02b09";
            isDefault = true;
            version = 1;
          }
          {
            inherit (cfg) orgId;
            name = "Loki";
            type = "loki";
            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
            uid = "cd6cc53e-840c-484d-85f7-96fede324006";
            version = 1;
          }
        ];
        deleteDatasources = [
          {
            inherit (cfg) orgId;
            name = "Prometheus";
          }
          {
            inherit (cfg) orgId;
            name = "Loki";
          }
        ];
      };
      alerting.contactPoints.settings = {
        apiVersion = 1;
        contactPoints = [{
          inherit (cfg) orgId;
          name = "grafana-default-email";
          receivers = lib.optionals ((builtins.length cfg.contactPoints) > 0) [{
            uid = "sysadmin";
            type = "email";
            settings.addresses = lib.concatStringsSep ";" cfg.contactPoints;
          }];
        }];
      };
      alerting.policies.settings = {
        apiVersion = 1;
        policies = [{
          inherit (cfg) orgId;
          receiver = "grafana-default-email";
          group_by = [ "grafana_folder" "alertname" ];
          group_wait = "30s";
          group_interval = "5m";
          repeat_interval = "4h";
        }];
        # resetPolicies seems to happen after setting the above policies, effectively rolling back
        # any updates.
      };
      alerting.rules.settings =
        let
          rules = builtins.fromJSON (builtins.readFile ./monitoring/rules.json);
          ruleIds = map (r: r.uid) rules;
        in
          {
            apiVersion = 1;
            groups = [{
              inherit (cfg) orgId;
              name = "SysAdmin";
              folder = "Self Host Blocks";
              interval = "10m";
              inherit rules;
            }];
            # deleteRules seems to happen after creating the above rules, effectively rolling back
            # any updates.
          };
    };

    services.prometheus = {
      enable = true;
      port = cfg.prometheusPort;
    };

    services.loki = {
      enable = true;
      dataDir = "/var/lib/loki";
      package = if cfg.lokiMajorVersion == 3 then pkgs.grafana-loki else
        # Comes from https://github.com/NixOS/nixpkgs/commit/8f95320f39d7e4e4a29ee70b8718974295a619f4
        (pkgs.grafana-loki.overrideAttrs (finalAttrs: previousAttrs: rec {
          version = "2.9.6";

          src = pkgs.fetchFromGitHub {
            owner = "grafana";
            repo = "loki";
            rev = "v${version}";
            hash = "sha256-79hK7axHf6soku5DvdXkE/0K4WKc4pnS9VMbVc1FS2I=";
          };

          subPackages = [
            "cmd/loki"
            "cmd/loki-canary"
            "clients/cmd/promtail"
            "cmd/logcli"
            # Removes "cmd/lokitool"
          ];

          ldflags = let t = "github.com/grafana/loki/pkg/util/build"; in [
            "-s"
            "-w"
            "-X ${t}.Version=${version}"
            "-X ${t}.BuildUser=nix@nixpkgs"
            "-X ${t}.BuildDate=unknown"
            "-X ${t}.Branch=unknown"
            "-X ${t}.Revision=unknown"
          ];
        }));
      configuration = {
        auth_enabled = false;

        server.http_listen_port = cfg.lokiPort;

        ingester = {
          lifecycler = {
            address = "127.0.0.1";
            ring = {
              kvstore.store = "inmemory";
              replication_factor = 1;
            };
            final_sleep = "0s";
          };
          chunk_idle_period = "5m";
          chunk_retain_period = "30s";
        };

        schema_config = {
          configs = [
            {
              from = "2018-04-15";
              store = "boltdb";
              object_store = "filesystem";
              schema = "v9";
              index.prefix = "index_";
              index.period = "168h";
            }
          ];
        };

        storage_config = {
          boltdb.directory = "/tmp/loki/index";
          filesystem.directory = "/tmp/loki/chunks";
        };

        limits_config = {
          enforce_metric_name = false;
          reject_old_samples = true;
          reject_old_samples_max_age = "168h";
        };

        chunk_store_config = {
          max_look_back_period = 0;
        };

        table_manager = {
          chunk_tables_provisioning = {
            inactive_read_throughput = 0;
            inactive_write_throughput = 0;
            provisioned_read_throughput = 0;
            provisioned_write_throughput = 0;
          };
          index_tables_provisioning = {
            inactive_read_throughput = 0;
            inactive_write_throughput = 0;
            provisioned_read_throughput = 0;
            provisioned_write_throughput = 0;
          };
          retention_deletes_enabled = false;
          retention_period = 0;
        };
      };
    };

    services.promtail = {
      enable = true;
      configuration = {
        server = {
          http_listen_port = 9080;
          grpc_listen_port = 0;
        };

        positions.filename = "/tmp/positions.yaml";

        client.url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}/api/prom/push";

        scrape_configs = [
          {
            job_name = "systemd";
            journal = {
              json = false;
              max_age = "12h";
              path = "/var/log/journal";
              # matches = "_TRANSPORT=kernel";
              labels = {
                job = "systemd-journal";
              };
            };
            relabel_configs = [
              {
                source_labels = [ "__journal__systemd_unit" ];
                target_label = "unit";
              }
            ];
          }
        ];
      };
    };

    services.nginx = {
      enable = true;

      virtualHosts.${fqdn} = {
        forceSSL = !(isNull cfg.ssl);
        sslCertificate = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.cert;
        sslCertificateKey = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.key;

        locations."/" = {
          proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
          proxyWebsockets = true;
          extraConfig = ''
          proxy_set_header Host $host;
          '';
        };
      };
    };

    services.prometheus.scrapeConfigs = [
      {
        job_name = "node";
        static_configs = [{
          targets = ["127.0.0.1:${toString config.services.prometheus.exporters.node.port}"];
        }];
      }
      {
        job_name = "netdata";
        metrics_path = "/api/v1/allmetrics";
        params.format = [ "prometheus" ];
        honor_labels = true;
        static_configs = [{
          targets = [ "127.0.0.1:19999" ];
        }];
      }
      {
        job_name = "smartctl";
        static_configs = [{
          targets = ["127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"];
        }];
      }
      {
        job_name = "prometheus_internal";
        static_configs = [{
          targets = ["127.0.0.1:${toString config.services.prometheus.port}"];
        }];
      }
    ] ++ (lib.lists.optional config.services.nginx.enable {
        job_name = "nginx";
        static_configs = [{
          targets = ["127.0.0.1:${toString config.services.prometheus.exporters.nginx.port}"];
        }];
    # }) ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
    #     job_name = "redis";
    #     static_configs = [
    #       {
    #         targets = ["127.0.0.1:${toString config.services.prometheus.exporters.redis.port}"];
    #       }
    #     ];
    # }) ++ (lib.optional (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
    #     job_name = "openvpn";
    #     static_configs = [
    #       {
    #         targets = ["127.0.0.1:${toString config.services.prometheus.exporters.openvpn.port}"];
    #       }
    #     ];
    }) ++ (lib.optional config.services.dnsmasq.enable {
        job_name = "dnsmasq";
        static_configs = [{
          targets = ["127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}"];
        }];
    });
    services.prometheus.exporters.nginx = lib.mkIf config.services.nginx.enable {
      enable = true;
      port = 9111;
      listenAddress = "127.0.0.1";
      scrapeUri = "http://localhost:80/nginx_status";
    };
    services.prometheus.exporters.node = {
      enable = true;
      # https://github.com/prometheus/node_exporter#collectors
      enabledCollectors = ["ethtool"];
      port = 9112;
      listenAddress = "127.0.0.1";
    };
    # https://github.com/nixos/nixpkgs/commit/12c26aca1fd55ab99f831bedc865a626eee39f80
    # TODO: remove when https://github.com/NixOS/nixpkgs/pull/205165 is merged
    services.udev.extraRules = ''
      SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk"
    '';
    services.prometheus.exporters.smartctl = {
      enable = true;
      port = 9115;
      listenAddress = "127.0.0.1";
    };
    # services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
    #   enable = true;
    #   port = 9119;
    #   listenAddress = "127.0.0.1";
    # };
    # services.prometheus.exporters.openvpn = lib.mkIf (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
    #   enable = true;
    #   port = 9121;
    #   listenAddress = "127.0.0.1";
    #   statusPaths = lib.mapAttrsToList (name: _config: "/tmp/openvpn/${name}.status") config.services.openvpn.servers;
    # };
    services.prometheus.exporters.dnsmasq = lib.mkIf config.services.dnsmasq.enable {
      enable = true;
      port = 9211;
      listenAddress = "127.0.0.1";
    };
    services.nginx.statusPage = lib.mkDefault config.services.nginx.enable;
    services.netdata = {
      enable = true;
      config = {
        # web.mode = "none";
        # web."bind to" = "127.0.0.1:19999";
        global = {
          "debug log" = "syslog";
          "access log" = "syslog";
          "error log" = "syslog";
        };
      };
    };
  };
}