From d2066525c9e6e4fd64f377a6b408fd007fae3aec Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 4 Feb 2024 18:05:05 +0200 Subject: [PATCH] Add support for Grafana Loki and Promtail Most of the work on these roles and integration was done by https://github.com/moan0s and https://github.com/sudo-Tiz --- docs/services/grafana-loki.md | 69 ++++++++++++ docs/services/grafana.md | 37 ++++++- docs/services/promtail.md | 171 ++++++++++++++++++++++++++++++ docs/supported-services.md | 2 + templates/group_vars_mash_servers | 103 ++++++++++++++++++ templates/requirements.yml | 8 ++ templates/setup.yml | 8 ++ 7 files changed, 397 insertions(+), 1 deletion(-) create mode 100644 docs/services/grafana-loki.md create mode 100644 docs/services/promtail.md diff --git a/docs/services/grafana-loki.md b/docs/services/grafana-loki.md new file mode 100644 index 0000000..b2a1649 --- /dev/null +++ b/docs/services/grafana-loki.md @@ -0,0 +1,69 @@ +# Grafana Loki + +[Grafana Loki](https://grafana.com/docs/loki/latest/) is a set of components that can be composed into a fully featured logging stack. Installing it is powered by the [mother-of-all-self-hosting/ansible-role-loki](https://github.com/mother-of-all-self-hosting/ansible-role-loki) Ansible role. + +Loki is just a log storage system. In order to make use of it, you'd need at least 2 other components + +- some agent (like [Promtail](./promtail.md)) to send logs to Loki +- some system (like [Grafana](./grafana.md)) to read the logs out of Loki and display them nicely + + +## Dependencies + +This service requires the following other services: + +- (optionally) [Traefik](traefik.md) - a reverse-proxy server for exposing Loki publicly +- (optionally) [Promtail](./promtail.md) - an agent that can send logs to Loki +- (optionally) [Grafana](./grafana.md) - a web UI that can query the Loki datasource (connection) and display the logs + + +## Configuration + +To enable this service, add the following configuration to your `vars.yml` file and re-run the [installation](../installing.md) process: + +```yaml +######################################################################## +# # +# loki # +# # +######################################################################## + +loki_enabled: true + +######################################################################## +# # +# /loki # +# # +######################################################################## +``` + +### Exposing the web interface + +By setting a hostname and optionally a path prefix, you can expose Loki publicly. You may wish to do this, if you'd like to be able to: + +- push logs from remote agents (e.g. Promtail installed on remote machines, etc.) +- query logs from remote systems (e.g. Grafana installed elsewhere) + +When exposing publicly, it's natural to set up [HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication) **or anyone would be able to read your logs or push new ones**. + +```yaml +loki_hostname: mash.example.com +loki_path_prefix: /loki + +# If you are sure you wish to run without Basic Auth enabled, +# explicitly set the variable below to false. +loki_container_labels_middleware_basic_auth_enabled: true +# Use `htpasswd -nb USERNAME PASSSWORD` to generate the users below. +loki_container_labels_middleware_basic_auth_users: '' +``` + + +## Usage + +After [installing](../installing.md), refer to the [official documentation](https://grafana.com/docs/loki/latest/reference/api/#post-lokiapiv1push) to send logs to loki's API without an agent or set up one or more instances of the [Promtail](./promtail.md) agent. + + +## Recommended other services + +- [Grafana](grafana.md) - a web-based tool for visualizing your Promtail logs (stored in [Grafana Loki](grafana-loki.md) or elsewhere) +- [Promtail](promtail.md) - an agent which ships the contents of local logs to a private [Grafana Loki](services/grafana-loki.md) instance diff --git a/docs/services/grafana.md b/docs/services/grafana.md index 94b4481..3af5bdc 100644 --- a/docs/services/grafana.md +++ b/docs/services/grafana.md @@ -47,6 +47,9 @@ Grafana is merely a visualization tool. It needs to pull data from a metrics (ti You can add multiple data sources to Grafana. +Below, we show a few examples of connecting Grafana to local datasources (running in containers on the same machine). +**If you're enabling multiple, you need to "merge" the configurations**. That is, don't define `grafana_provisioning_datasources` or `grafana_container_additional_networks_custom` twice, but combine them. + #### Integrating with a local Prometheus instance If you're installing [Prometheus](prometheus.md) on the same server, you can hook Grafana to it over the container network with the following **additional** configuration: @@ -57,14 +60,46 @@ grafana_provisioning_datasources: type: prometheus access: proxy url: "http://{{ prometheus_identifier }}:9090" + # Enable below if connecting to a remote instance that uses Basic Auth. + # basicAuth: true + # basicAuthUser: loki + # secureJsonData: + # basicAuthPassword: "" # Prometheus runs in another container network, so we need to connect to it. grafana_container_additional_networks_custom: - "{{ prometheus_container_network }}" ``` -For connecting to a **remote** Prometheus instance, you may need to adjust this configuration somehow. +For connecting to a **remote** Prometheus instance, you may need to adjust this configuration. +#### Integrating with a local Loki instance + +If you're installing [Grafana Loki](grafana-loki.md) on the same server, you can hook Grafana to it over the container network with the following **additional** configuration: + +```yaml +grafana_provisioning_datasources: + - name: Loki (your-tenant-id) + type: loki + access: proxy + url: "http://{{ loki_identifier }}:{{ loki_server_http_listen_port }}" + # Enable below and also (basicAuthPassword) if connecting to a remote instance that uses Basic Auth. + # basicAuth: true + # basicAuthUser: loki + jsonData: + httpHeaderName1: X-Scope-OrgID + secureJsonData: + httpHeaderValue1: "your-tenant-id" + # basicAuthPassword: "" + +# Loki runs in another container network, so we need to connect to it. +grafana_container_additional_networks_custom: + - "{{ loki_container_network }}" +``` + +For connecting to a **remote** Loki instance, you may need to adjust this configuration. + +If you're installing [Promtail](./promtail.md) on the same server as Loki, by default it's configured to send `mash` as the tenant ID. ### Integrating with Prometheus Node Exporter diff --git a/docs/services/promtail.md b/docs/services/promtail.md new file mode 100644 index 0000000..2d26c96 --- /dev/null +++ b/docs/services/promtail.md @@ -0,0 +1,171 @@ +# Promtail + +[Promtail](https://grafana.com/oss/promtail/) agent is a log aggregation system designed to store and query logs from all your applications and infrastructure. It integrates nicely with [Grafana Loki](./grafana-loki.md). + + +## Dependencies + +This service requires the following other services: + +- [Grafana Loki](grafana-loki.md) - a log-storage server where you'd be sending the logs +- (optional) [Traefik](traefik.md) - a reverse-proxy server, if you're exposing Promtail's metrics or API + + +## Configuration + +To enable this service, add the following configuration to your `vars.yml` file and re-run the [installation](../installing.md) process: + +```yaml +######################################################################## +# # +# promtail # +# # +######################################################################## + +promtail_enabled: true + +# See "Configuring scrapers" below. +# You need to enable at least one scraper to have Promtail do anything. + +# If you haven't enabled Grafana Loki on the same server, you will need +# to define some clients to push logs to. +# See "Configuring clients" below. + +######################################################################## +# # +# /promtail # +# # +######################################################################## +``` + +### Configuring scrapers + +**No scrapers are enabled by default**. As such, Promtail does not do anything in its default configuration. + +Below, we show you a few built-in scrapers you can easily enable, as well as how to create your own custom ones. + +#### Scraping systemd-journald logs + +To scrape the [systemd Journal](https://wiki.archlinux.org/title/Systemd/Journal), enable the already-prepared scraper for this with this additional `vars.yml` configuration: + +```yml +# Some distros only store a non-persistent (in-memory) journal in a path like in `/run/log/journal`. +# Others may be using a path different than `/var/log/journal`. +# Adjust accordingly. +promtail_journald_scraper_enabled: true +promtail_journald_scraper_host_path: /var/log/journal +``` + +#### Scraping textual log files (/var/log, etc.) + +A lot of distros dump textual log files in `/var/log`. To scrape them, enable the already-prepared scraper for this with this additional `vars.yml` configuration: + +```yml +promtail_varlog_scraper_enabled: true +# Consider adjusting this if you'd like to scrape a different path +# promtail_varlog_scraper_host_path: /var/log +``` + +#### Scraping other directories + +Besides the predefined scrapers described above, you can also define your own additional ones with the help of these variables: + +- `promtail_container_additional_mounts_custom`, to mount additional paths into the Promtail container +- `promtail_config_scrape_configs_custom`, to inject additional jobs into Promtail's `scrape_configs` configuration. See `promtail_journald_scraper_config` and `promtail_varlog_scraper_config` for an example + +Here's an example for scraping some hypothethical SSH logs stored somewhere: + +```yml +promtail_container_additional_mounts_custom: + - "type=bind,source=,target=/data/ssh,readonly" + + +promtail_config_scrape_configs_custom: + - job_name: ssh + static_configs: + - localhost + __path__: /data/ssh + labels: + job: ssh +``` + +##### Scraping syslog + +The following example demonstrates the use of rsyslog and promtail to scrape syslog logs. + +**Prerequisites**: Edit your rsyslog configuration in order to send logs to `promtail.*`` +This could be done by creating a `/etc/rsyslog.d/00-promtail-relay.conf` file with the following content: + +``` +*.* action(type="omfwd" protocol="tcp" target="" port="" Template="RSYSLOG_SyslogProtocol23Format" TCP_Framing="octet-counted" KeepAlive="on") +``` + +The port is a port number that you come up with yourself (e.g. `1234`). + +First, you need a custom scrape configuration which tells Promtail to listen on this port (replace `SOME_PORT_NUMBER_IN_CONTAINER` with your port number of choice): + +```yaml +promtail_config_scrape_configs_custom: + - job_name: syslog + syslog: + listen_address: 0.0.0.0:SOME_PORT_NUMBER_IN_CONTAINER + labels: + job: syslog + relabel_configs: + - source_labels: [__syslog_message_hostname] + target_label: host + - source_labels: [__syslog_message_hostname] + target_label: hostname + - source_labels: [__syslog_message_severity] + target_label: level + - source_labels: [__syslog_message_app_name] + target_label: application + - source_labels: [__syslog_message_facility] + target_label: facility + - source_labels: [__syslog_connection_hostname] + target_label: connection_hostname +``` + +You'd then need to expose this TCP port outside of the container, so that the local host (or remote host) can reach it. + +To expose it on the loopback interface (reachable only from the same machine), use a configuration like this: +```yaml +promtail_container_extra_arguments_custom: + - "-p 127.0.0.1:1234:1234" +``` + + +### Configuring clients + +If you've also enabled [Grafana Loki](./grafana-loki.md) on the same server, Promtail will automatically be configured to push logs to it. + +Otherwise, you will need to extend the Promtail configuration by specifying clients to push to. Add something like this to your `vars.yml` configuration: + +```yml +promtail_config_clients_custom: + # Note the double /loki/loki. + # This assumes Loki is installed at a `/loki` path-prefix. + - url: https://mash.example.com/loki/loki/api/v1/push +``` + +### Exposing the web interface + +There are 2 reasons to expose Promtail to the public web: + +1. So that you can scrape its Prometheus-compatible `/metrics` endpoint or observe its current `/targets` via API +2. So that you can use [loki_push_api](https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#loki_push_api) and push logs to Promtail (so that it can forward them onto its [clients](#configuring-clients)). This feature likely needs to be enabled explicitly. + +To expose Promtail to the web, you need to assign a hostname in `promtail_hostname` and optionally a path-prefix. + +You can then decide whether you'd like to expose Promtail's whole API via `promtail_container_labels_api_enabled` or just its metrics endpoint via `promtail_container_labels_metrics_enabled`. + +Consult the `defaults/main.yml` file for variables related to these. + +When exposing metrics, and especially the whole API, it's important to protected them. The Promtail Ansible role has variables that let you easily set up [HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication) via `promtail_container_labels_api_traefik_middleware_basic_auth_*` and `promtail_container_labels_metrics_traefik_middleware_basic_auth_*` variables. + + + +## Recommended other services + +- [Grafana Loki](grafana-loki.md) - a storage server for your logs compatible with Promtail +- [Grafana](grafana.md) - a web-based tool for visualizing your Promtail logs (stored in [Grafana Loki](grafana-loki.md) or elsewhere) diff --git a/docs/supported-services.md b/docs/supported-services.md index c745cfd..6f7067e 100644 --- a/docs/supported-services.md +++ b/docs/supported-services.md @@ -27,6 +27,7 @@ | [Gitea](https://gitea.io/) | A painless self-hosted [Git](https://git-scm.com/) service. | [Link](services/gitea.md) | | [GoToSocial](https://gotosocial.org/) | A self-hosted [ActivityPub](https://activitypub.rocks/) social network server | [Link](services/gotosocial.md) | | [Grafana](https://grafana.com/) | An open and composable observability and data visualization platform, often used with [Prometheus](services/prometheus.md) | [Link](services/grafana.md) | +| [Grafana Loki](https://grafana.com/docs/loki/latest/) | Open-source log aggregation system that helps collect, store, and analyze logs in a scalable and efficient manner | [Link](services/grafana-loki.md) | | [Healthchecks](https://healthchecks.io/) | A simple and Effective Cron Job Monitoring solution | [Link](services/healthchecks.md) | | [Hubsite](https://github.com/moan0s/hubsite) | A simple, static site that shows an overview of the available services | [Link](services/hubsite.md) | | [ILMO](https://github.com/moan0s/ILMO2) | An open source library management tool. | [Link](services/ilmo.md) | @@ -58,6 +59,7 @@ | [Prometheus SSH Exporter](https://github.com/treydock/ssh_exporter) | SSH probes | [Link](services/prometheus-ssh-exporter.md) | | [Prometheus Node Exporter](https://github.com/prometheus/node_exporter) | Exporter for machine metrics | [Link](services/prometheus-node-exporter.md) | | [Prometheus Postgres Exporter](https://github.com/prometheus-community/postgres_exporter) | A PostgreSQL metric exporter for Prometheus | [Link](services/prometheus-postgres-exporter.md) | +| [Promtail](https://grafana.com/docs/loki/latest/send-data/promtail/) | An agent which ships the contents of local logs to a private [Grafana Loki](services/grafana-loki.md) instance | [Link](services/promtail.md) | | [Radicale](https://radicale.org/) | A Free and Open-Source CalDAV and CardDAV Server (solution for hosting contacts and calendars) | [Link](services/radicale.md) | | [Redmine](https://redmine.org/) | A flexible project management web application. | [Link](services/redmine.md) | | [Redis](https://redis.io/) | An in-memory data store used by millions of developers as a database, cache, streaming engine, and message broker. | [Link](services/redis.md) | diff --git a/templates/group_vars_mash_servers b/templates/group_vars_mash_servers index 7b24ee2..cfa171a 100644 --- a/templates/group_vars_mash_servers +++ b/templates/group_vars_mash_servers @@ -362,6 +362,11 @@ mash_playbook_devture_systemd_service_manager_services_list_auto_itemized: {{ ({'name': (lago_identifier + '-pdf.service'), 'priority': 1900, 'groups': ['mash', 'lago', 'lago-pdf']} if lago_enabled else omit) }} # /role-specific:lago + # role-specific:loki + - |- + {{ ({'name': (loki_identifier + '.service'), 'priority': 2000, 'groups': ['mash', 'loki']} if loki_enabled else omit) }} + # /role-specific:loki + # role-specific:linkding - |- {{ ({'name': (linkding_identifier + '.service'), 'priority': 2000, 'groups': ['mash', 'linkding']} if linkding_enabled else omit) }} @@ -473,6 +478,11 @@ mash_playbook_devture_systemd_service_manager_services_list_auto_itemized: {{ ({'name': (prometheus_postgres_exporter_identifier + '.service'), 'priority': 500, 'groups': ['mash', 'metrics', 'prometheus-postgres-exporter']} if prometheus_postgres_exporter_enabled else omit) }} # /role-specific:prometheus_postgres_exporter + # role-specific:promtail + - |- + {{ ({'name': (promtail_identifier + '.service'), 'priority': 500, 'groups': ['mash', 'logs', 'promtail']} if promtail_enabled else omit) }} + # /role-specific:promtail + # role-specific:radicale - |- {{ ({'name': (radicale_identifier + '.service'), 'priority': 2000, 'groups': ['mash', 'radicale']} if radicale_enabled else omit) }} @@ -3017,6 +3027,43 @@ lago_api_environment_variable_encryption_key_derivation_salt: "{{ '%s' | format( # /role-specific:lago + +# role-specific:loki +######################################################################## +# # +# loki # +# # +######################################################################## + +loki_enabled: false + +loki_identifier: "{{ mash_playbook_service_identifier_prefix }}loki" + +loki_base_path: "{{ mash_playbook_base_path }}/{{ mash_playbook_service_base_directory_name_prefix }}loki" + +loki_uid: "{{ mash_playbook_uid }}" +loki_gid: "{{ mash_playbook_gid }}" + +# Only enable Traefik labels if a hostname is set (indicating that this will be exposed publicly) +loki_container_labels_traefik_enabled: "{{ mash_playbook_traefik_labels_enabled and loki_hostname | length > 0 }}" +loki_container_labels_traefik_docker_network: "{{ mash_playbook_reverse_proxyable_services_additional_network }}" +loki_container_labels_traefik_entrypoints: "{{ devture_traefik_entrypoint_primary }}" +loki_container_labels_traefik_tls_certResolver: "{{ devture_traefik_certResolver_primary }}" + +loki_container_additional_networks_auto: | + {{ + ([mash_playbook_reverse_proxyable_services_additional_network] if mash_playbook_reverse_proxyable_services_additional_network else []) + }} + +######################################################################## +# # +# /loki # +# # +######################################################################## +# /role-specific:loki + + + # role-specific:linkding ######################################################################## # # @@ -3064,6 +3111,7 @@ linkding_database_engine: "{{ 'postgres' if devture_postgres_enabled and linkdin # /role-specific:linkding + # role-specific:miniflux ######################################################################## # # @@ -3950,6 +3998,61 @@ prometheus_node_exporter_container_extra_arguments: +# role-specific:promtail +######################################################################## +# # +# promtail # +# # +######################################################################## + +promtail_enabled: false + +promtail_identifier: "{{ mash_playbook_service_identifier_prefix }}promtail" + +promtail_base_path: "{{ mash_playbook_base_path }}/{{ mash_playbook_service_base_directory_name_prefix }}promtail" + +promtail_uid: "{{ mash_playbook_uid }}" +promtail_gid: "{{ mash_playbook_gid }}" + +# Only enable Traefik labels if a hostname is set (indicating that this will be exposed publicly) +promtail_container_labels_traefik_enabled: "{{ mash_playbook_traefik_labels_enabled and promtail_hostname | length > 0 }}" +promtail_container_labels_traefik_docker_network: "{{ mash_playbook_reverse_proxyable_services_additional_network }}" +promtail_container_labels_traefik_entrypoints: "{{ devture_traefik_entrypoint_primary }}" +promtail_container_labels_traefik_tls_certResolver: "{{ devture_traefik_certResolver_primary }}" + +# role-specific:loki +promtail_config_clients_auto: | + {{ + ([{ + 'url': ('http://' + loki_identifier + ':' + (loki_server_http_listen_port | string) + '/loki/api/v1/push'), + 'tenant_id': 'mash', + }] if loki_enabled else []) + }} +# /role-specific:loki + +promtail_container_labels_metrics_enabled: "{{ prometheus_enabled | default(false) or mash_playbook_metrics_exposure_enabled }}" +promtail_container_labels_metrics_hostname: "{{ mash_playbook_metrics_exposure_hostname }}" +promtail_container_labels_metrics_path_prefix: "{{ mash_playbook_metrics_exposure_path_prefix }}/{{ promtail_identifier }}" +promtail_container_labels_metrics_traefik_middleware_basic_auth_enabled: "{{ mash_playbook_metrics_exposure_http_basic_auth_enabled }}" +promtail_container_labels_metrics_traefik_middleware_basic_auth_users: "{{ mash_playbook_metrics_exposure_http_basic_auth_users }}" + +promtail_container_additional_networks_auto: | + {{ + ( + ([mash_playbook_reverse_proxyable_services_additional_network] if mash_playbook_reverse_proxyable_services_additional_network else []) + + + ([loki_container_network] if (loki_enabled | default(false) and loki_container_network | default('') != promtail_container_network) else []) + ) | unique + }} + +######################################################################## +# # +# /promtail # +# # +######################################################################## +# /role-specific:promtail + + # role-specific:radicale ######################################################################## # # diff --git a/templates/requirements.yml b/templates/requirements.yml index b3b6c15..adae0e4 100644 --- a/templates/requirements.yml +++ b/templates/requirements.yml @@ -152,6 +152,10 @@ version: v0.50.0-0 name: lago activation_prefix: lago_ +- src: git+https://github.com/mother-of-all-self-hosting/ansible-role-loki.git + version: v2.9.4-3 + name: loki + activation_prefix: loki_ - src: git+https://github.com/kinduff/ansible-docker-linkding.git version: v1.9.0 name: linkding @@ -256,6 +260,10 @@ version: v1.5.0-2 name: prometheus_ssh_exporter activation_prefix: prometheus_ssh_exporter_ +- src: git+https://github.com/mother-of-all-self-hosting/ansible-role-promtail.git + version: v2.9.4-0 + name: promtail + activation_prefix: promtail_ - src: git+https://gitlab.com/etke.cc/roles/radicale.git version: v3.1.8.3-1 name: radicale diff --git a/templates/setup.yml b/templates/setup.yml index 1d7c10e..0290749 100644 --- a/templates/setup.yml +++ b/templates/setup.yml @@ -234,6 +234,10 @@ - role: galaxy/linkding # /role-specific:linkding + # role-specific:loki + - role: galaxy/loki + # /role-specific:loki + # role-specific:mobilizon - role: galaxy/mobilizon # /role-specific:mobilizon @@ -294,6 +298,10 @@ - role: galaxy/prometheus_ssh_exporter # /role-specific:prometheus_ssh_exporter + # role-specific:promtail + - role: galaxy/promtail + # /role-specific:promtail + # role-specific:radicale - role: galaxy/radicale # /role-specific:radicale