From 35ae4481f2535a7490034a0620059631d7428ebb Mon Sep 17 00:00:00 2001
From: ibizaman <ibizapeanut@gmail.com>
Date: Sun, 26 Nov 2023 09:49:40 -0800
Subject: [PATCH] move all monitoring documentation to the docs/ folder

---
 README.md                 | 38 +----------------------
 docs/blocks/monitoring.md | 65 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index 0178285..2657c11 100644
--- a/README.md
+++ b/README.md
@@ -326,43 +326,7 @@ See the [`ldap.nix`](./modules/ldap.nix) and [`authelia.nix`](./modules/authelia
 
 ### Deploy the full Grafana, Prometheus and Loki suite
 
-This is not a prerequisite for anything and could be enabled just for debugging.
-
-```nix
-shb.monitoring = {
-  enable = true;
-  subdomain = "grafana";
-  inherit domain;
-};
-```
-
-With that, Grafana, Prometheus, Loki and Promtail are setup! You can access `Grafana` at
-`grafana.example.com`.
-
-A few Prometheus metrics scrapers are setup automatically:
-- node - cpu, memory, disk I/O, network I/O and a few others of the computer
-- smartctl - hard drive health
-- prometheus_internal - scraping jobs health
-- nginx
-- dnsmasq (if the service is enabled)
-
-The following Loki logs scraper is setup automatically:
-- systemd journal
-
-I intend to provide more options so that you could for example tweak data retention.
-
-Also, since all logs are now stored in Loki, you can probably reduce the systemd journal retention
-time with:
-
-```nix
-# See https://www.freedesktop.org/software/systemd/man/journald.conf.html#SystemMaxUse=
-services.journald.extraConfig = ''
-SystemMaxUse=2G
-SystemKeepFree=4G
-SystemMaxFileSize=100M
-MaxFileSec=day
-'';
-```
+See [docs/blocks/monitoring.md](docs/blocks/monitoring.md).
 
 ### Set up network tunnel with VPN and Proxy
 
diff --git a/docs/blocks/monitoring.md b/docs/blocks/monitoring.md
index 2f93f4b..7a26ebb 100644
--- a/docs/blocks/monitoring.md
+++ b/docs/blocks/monitoring.md
@@ -6,6 +6,71 @@ This block sets up the monitoring stack for Self Host Blocks. It is composed of:
 - Prometheus as the database for metrics.
 - Loki as the database for logs.
 
+## Configuration
+
+```nix
+shb.monitoring = {
+  enable = true;
+  subdomain = "grafana";
+  inherit domain;
+  contactPoints = [ "me@example.com" ];
+  adminPasswordFile = config.sops.secrets."monitoring/admin_password".path;
+  secretKeyFile = config.sops.secrets."monitoring/secret_key".path;
+};
+
+sops.secrets."monitoring/admin_password" = {
+  sopsFile = ./secrets.yaml;
+  mode = "0400";
+  owner = "grafana";
+  group = "grafana";
+  restartUnits = [ "grafana.service" ];
+};
+sops.secrets."monitoring/secret_key" = {
+  sopsFile = ./secrets.yaml;
+  mode = "0400";
+  owner = "grafana";
+  group = "grafana";
+  restartUnits = [ "grafana.service" ];
+};
+```
+
+With that, Grafana, Prometheus, Loki and Promtail are setup! You can access `Grafana` at
+`grafana.example.com` with user `admin` and password ``.
+
+I recommend adding a STMP server configuration so you receive alerts by email:
+
+```nix
+shb.monitoring.smtp = {
+  from_address = "grafana@$example.com";
+  from_name = "Grafana";
+  host = "smtp.mailgun.org";
+  port = 587;
+  username = "postmaster@mg.example.com";
+  passwordFile = config.sops.secrets."monitoring/smtp".path;
+};
+
+sops.secrets."monitoring/secret_key" = {
+  sopsFile = ./secrets.yaml;
+  mode = "0400";
+  owner = "grafana";
+  group = "grafana";
+  restartUnits = [ "grafana.service" ];
+};
+```
+
+Since all logs are now stored in Loki, you can probably reduce the systemd journal retention
+time with:
+
+```nix
+# See https://www.freedesktop.org/software/systemd/man/journald.conf.html#SystemMaxUse=
+services.journald.extraConfig = ''
+SystemMaxUse=2G
+SystemKeepFree=4G
+SystemMaxFileSize=100M
+MaxFileSec=day
+'';
+```
+
 ## Provisioning
 
 Self Host Blocks will create automatically the following resources: