From 6aed5ee6a5547c3aa18f9afa531a642a465511d8 Mon Sep 17 00:00:00 2001 From: ibizaman Date: Tue, 20 Aug 2024 07:09:10 +0200 Subject: [PATCH] add backup contract --- docs/blocks.md | 4 +- docs/contracts.md | 9 +- docs/default.nix | 13 +- flake.lock | 54 +--- flake.nix | 4 +- modules/blocks/{backup.nix => borgbackup.nix} | 105 ++------ modules/blocks/restic.nix | 238 ++++++++++++++++++ .../blocks/{backup => restic}/docs/default.md | 116 ++++----- modules/contracts/backup.nix | 61 +++++ modules/contracts/backup/docs/default.md | 56 +++++ modules/contracts/backup/dummyModule.nix | 10 + modules/contracts/default.nix | 1 + modules/services/vaultwarden.nix | 32 ++- modules/services/vaultwarden/docs/default.md | 22 ++ test/blocks/restic.nix | 156 ++++++++++++ test/common.nix | 37 +-- 16 files changed, 685 insertions(+), 233 deletions(-) rename modules/blocks/{backup.nix => borgbackup.nix} (69%) create mode 100644 modules/blocks/restic.nix rename modules/blocks/{backup => restic}/docs/default.md (60%) create mode 100644 modules/contracts/backup.nix create mode 100644 modules/contracts/backup/docs/default.md create mode 100644 modules/contracts/backup/dummyModule.nix create mode 100644 test/blocks/restic.nix diff --git a/docs/blocks.md b/docs/blocks.md index 3849fd2..9eff596 100644 --- a/docs/blocks.md +++ b/docs/blocks.md @@ -34,8 +34,8 @@ Not all blocks are yet documented. You can find all available blocks [in the rep modules/blocks/ssl/docs/default.md ``` -```{=include=} chapters html:into-file=//blocks-backup.html -modules/blocks/backup/docs/default.md +```{=include=} chapters html:into-file=//blocks-restic.html +modules/blocks/restic/docs/default.md ``` ```{=include=} chapters html:into-file=//blocks-monitoring.html diff --git a/docs/contracts.md b/docs/contracts.md index 8af45dc..d241b90 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -19,12 +19,19 @@ as possible, reducing the quite thick layer that it is now. Provided contracts are: -- [SSL generator contract](contracts-ssl.html) to generate SSL certificates. Two implementations are provided: self-signed and Let's Encrypt. +- [SSL generator contract](contracts-ssl.html) to generate SSL certificates. + Two implementations are provided: self-signed and Let's Encrypt. +- [Backup contract](contracts-backup.html) to backup directories. + This contract allows to backup multiple times the same directories for extra protection. ```{=include=} chapters html:into-file=//contracts-ssl.html modules/contracts/ssl/docs/default.md ``` +```{=include=} chapters html:into-file=//contracts-backup.html +modules/contracts/backup/docs/default.md +``` + ## Why do we need this new concept? {#contracts-why} Currently in nixpkgs, every module needing access to a shared resource must implement the logic diff --git a/docs/default.nix b/docs/default.nix index e02ffe8..bfc6989 100644 --- a/docs/default.nix +++ b/docs/default.nix @@ -67,7 +67,9 @@ let }; optionsDocs = buildOptionsDocs { - modules = allModules ++ [ scrubbedModule ]; + modules = allModules ++ [ + scrubbedModule + ]; variablelistId = "selfhostblocks-options"; includeModuleSystemOptions = false; }; @@ -134,10 +136,10 @@ in stdenv.mkDerivation { '@OPTIONS_JSON@' \ ${individualModuleOptionsDocs ../modules/blocks/ssl.nix}/share/doc/nixos/options.json - substituteInPlace ./modules/blocks/backup/docs/default.md \ + substituteInPlace ./modules/blocks/restic/docs/default.md \ --replace \ '@OPTIONS_JSON@' \ - ${individualModuleOptionsDocs ../modules/blocks/backup.nix}/share/doc/nixos/options.json + ${individualModuleOptionsDocs ../modules/blocks/restic.nix}/share/doc/nixos/options.json substituteInPlace ./modules/services/nextcloud-server/docs/default.md \ --replace \ @@ -149,6 +151,11 @@ in stdenv.mkDerivation { '@OPTIONS_JSON@' \ ${individualModuleOptionsDocs ../modules/services/vaultwarden.nix}/share/doc/nixos/options.json + substituteInPlace ./modules/contracts/backup/docs/default.md \ + --replace \ + '@OPTIONS_JSON@' \ + ${individualModuleOptionsDocs ../modules/contracts/backup/dummyModule.nix}/share/doc/nixos/options.json + substituteInPlace ./modules/contracts/ssl/docs/default.md \ --replace \ '@OPTIONS_JSON@' \ diff --git a/flake.lock b/flake.lock index 08c9431..5431f5e 100644 --- a/flake.lock +++ b/flake.lock @@ -49,38 +49,6 @@ "type": "github" } }, - "nixpkgs-stable": { - "locked": { - "lastModified": 1716655032, - "narHash": "sha256-kQ25DAiCGigsNR/Quxm3v+JGXAEXZ8I7RAF4U94bGzE=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "59a450646ec8ee0397f5fa54a08573e8240eb91f", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "release-23.11", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1716651315, - "narHash": "sha256-iMgzIeedMqf30TXZ439zW3Yvng1Xm9QTGO+ZwG1IWSw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "c5187508b11177ef4278edf19616f44f21cc8c69", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, "nmdsrc": { "flake": false, "locked": { @@ -102,27 +70,7 @@ "flake-utils": "flake-utils", "nix-flake-tests": "nix-flake-tests", "nixpkgs": "nixpkgs", - "nmdsrc": "nmdsrc", - "sops-nix": "sops-nix" - } - }, - "sops-nix": { - "inputs": { - "nixpkgs": "nixpkgs_2", - "nixpkgs-stable": "nixpkgs-stable" - }, - "locked": { - "lastModified": 1716692524, - "narHash": "sha256-sALodaA7Zkp/JD6ehgwc0UCBrSBfB4cX66uFGTsqeFU=", - "owner": "Mic92", - "repo": "sops-nix", - "rev": "962797a8d7f15ed7033031731d0bb77244839960", - "type": "github" - }, - "original": { - "owner": "Mic92", - "repo": "sops-nix", - "type": "github" + "nmdsrc": "nmdsrc" } }, "systems": { diff --git a/flake.nix b/flake.nix index cb21bfb..fd69f8a 100644 --- a/flake.nix +++ b/flake.nix @@ -36,12 +36,12 @@ allModules = [ modules/blocks/authelia.nix - modules/blocks/backup.nix modules/blocks/davfs.nix modules/blocks/ldap.nix modules/blocks/monitoring.nix modules/blocks/nginx.nix modules/blocks/postgresql.nix + modules/blocks/restic.nix modules/blocks/ssl.nix modules/blocks/tinyproxy.nix modules/blocks/vpn.nix @@ -60,6 +60,7 @@ # Only used for documentation. contractDummyModules = [ + modules/contracts/backup/dummyModule.nix modules/contracts/ssl/dummyModule.nix ]; in @@ -133,6 +134,7 @@ // (vm_test "ldap" ./test/blocks/ldap.nix) // (vm_test "lib" ./test/blocks/lib.nix) // (vm_test "postgresql" ./test/blocks/postgresql.nix) + // (vm_test "restic" ./test/blocks/restic.nix) // (vm_test "ssl" ./test/blocks/ssl.nix) ); } diff --git a/modules/blocks/backup.nix b/modules/blocks/borgbackup.nix similarity index 69% rename from modules/blocks/backup.nix rename to modules/blocks/borgbackup.nix index 3e57d58..ebf0a04 100644 --- a/modules/blocks/backup.nix +++ b/modules/blocks/borgbackup.nix @@ -1,30 +1,32 @@ { config, pkgs, lib, utils, ... }: let - cfg = config.shb.backup; + cfg = config.shb.borgbackup; instanceOptions = { - enable = lib.mkEnableOption "shb backup instance"; - - backend = lib.mkOption { - description = "What program to use to make the backups."; - type = lib.types.enum [ "borgmatic" "restic" ]; - example = "borgmatic"; - }; + enable = lib.mkEnableOption "shb borgbackup"; keySopsFile = lib.mkOption { - description = "Sops file that holds this instance's Borgmatic repository key and passphrase."; + description = "Sops file that holds this instance's repository key and passphrase."; type = lib.types.path; example = "secrets/backup.yaml"; }; + encryptionKeyFile = lib.mkOption { + description = "Encryption key for the backup."; + type = lib.types.path; + }; + + encryption_passcommand = "cat /run/secrets/borgmatic/passphrases/${if isNull instance.secretName then name else instance.secretName}"; + borg_keys_directory = "/run/secrets/borgmatic/keys"; + sourceDirectories = lib.mkOption { - description = "Borgmatic source directories."; + description = "Source directories."; type = lib.types.nonEmptyListOf lib.types.str; }; excludePatterns = lib.mkOption { - description = "Borgmatic exclude patterns."; + description = "Exclude patterns."; type = lib.types.listOf lib.types.str; default = []; }; @@ -74,7 +76,7 @@ let }; consistency = lib.mkOption { - description = "Consistency frequency options. Only applicable for borgmatic"; + description = "Consistency frequency options."; type = lib.types.attrsOf lib.types.nonEmptyStr; default = {}; example = { @@ -84,7 +86,7 @@ let }; hooks = lib.mkOption { - description = "Borgmatic hooks."; + description = "Hooks to run before or after the backup."; default = {}; type = lib.types.submodule { options = { @@ -115,14 +117,7 @@ let in { - options.shb.backup = { - onlyOnAC = lib.mkOption { - description = "Run backups only if AC power is plugged in."; - default = true; - example = false; - type = lib.types.bool; - }; - + options.shb.borgbackup = { user = lib.mkOption { description = "Unix user doing the backups."; type = lib.types.str; @@ -163,12 +158,12 @@ in }; ioSchedulingClass = lib.mkOption { type = lib.types.enum [ "idle" "best-effort" "realtime" ]; - description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands."; + description = "ionice scheduling class, defaults to best-effort IO."; default = "best-effort"; }; ioPriority = lib.mkOption { type = lib.types.nullOr (lib.types.ints.between 0 7); - description = "ionice priority, defaults to 7 for lowest priority IO. Only used for `restic backup`, `restic forget` and `restic check` commands."; + description = "ionice priority, defaults to 7 for lowest priority IO."; default = 7; }; }; @@ -179,8 +174,6 @@ in config = lib.mkIf (cfg.instances != {}) ( let enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances; - borgmaticInstances = lib.attrsets.filterAttrs (k: i: i.backend == "borgmatic") enabledInstances; - resticInstances = lib.attrsets.filterAttrs (k: i: i.backend == "restic") enabledInstances; in lib.mkMerge [ # Secrets configuration { @@ -234,13 +227,13 @@ in } # Borgmatic configuration { - systemd.timers.borgmatic = lib.mkIf (borgmaticInstances != {}) { + systemd.timers.borgmatic = lib.mkIf (enabledInstances != {}) { timerConfig = { OnCalendar = "hourly"; }; }; - systemd.services.borgmatic = lib.mkIf (borgmaticInstances != {}) { + systemd.services.borgmatic = lib.mkIf (enabledInstances != {}) { serviceConfig = { User = cfg.user; Group = cfg.group; @@ -252,10 +245,10 @@ in }; }; - systemd.packages = lib.mkIf (borgmaticInstances != {}) [ pkgs.borgmatic ]; + systemd.packages = lib.mkIf (enabledInstances != {}) [ pkgs.borgmatic ]; environment.systemPackages = ( lib.optionals cfg.borgServer [ pkgs.borgbackup ] - ++ lib.optionals (borgmaticInstances != {}) [ pkgs.borgbackup pkgs.borgmatic ] + ++ lib.optionals (enabledInstances != {}) [ pkgs.borgbackup pkgs.borgmatic ] ); environment.etc = @@ -272,7 +265,7 @@ in }); storage = { - encryption_passcommand = "cat /run/secrets/borgmatic/passphrases/${if isNull instance.secretName then name else instance.secretName}"; + encryption_passcommand = "cat ${instance.encryptionKeyFile}"; borg_keys_directory = "/run/secrets/borgmatic/keys"; }; @@ -296,57 +289,7 @@ in }; }; in - lib.mkMerge (lib.attrsets.mapAttrsToList mkSettings borgmaticInstances); - } - # Restic configuration - { - environment.systemPackages = lib.optionals (resticInstances != {}) [ pkgs.restic ]; - - services.restic.backups = - let - mkRepositorySettings = name: instance: repository: { - "${name}_${repoSlugName repository.path}" = { - inherit (cfg) user; - repository = repository.path; - - paths = instance.sourceDirectories; - - passwordFile = "/run/secrets/${instance.backend}/passphrases/${name}"; - - initialize = true; - - inherit (repository) timerConfig; - - pruneOpts = lib.mapAttrsToList (name: value: - "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}" - ) instance.retention; - - backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup; - - backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup; - } // lib.attrsets.optionalAttrs (instance.environmentFile) { - environmentFile = "/run/secrets/${instance.backend}/environmentfiles/${name}"; - } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) { - exclude = instance.excludePatterns; - }; - }; - - mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; - in - lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings resticInstances)); - - systemd.services = - let - mkRepositorySettings = name: instance: repository: { - "restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = { - Nice = cfg.performance.niceness; - IOSchedulingClass = cfg.performance.ioSchedulingClass; - IOSchedulingPriority = cfg.performance.ioPriority; - }; - }; - mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; - in - lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings resticInstances)); + lib.mkMerge (lib.attrsets.mapAttrsToList mkSettings enabledInstances); } ]); } diff --git a/modules/blocks/restic.nix b/modules/blocks/restic.nix new file mode 100644 index 0000000..24014b3 --- /dev/null +++ b/modules/blocks/restic.nix @@ -0,0 +1,238 @@ +{ config, pkgs, lib, utils, ... }: + +let + cfg = config.shb.restic; + + shblib = pkgs.callPackage ../../lib {}; + + instanceOptions = { + enable = lib.mkEnableOption "shb restic"; + + passphraseFile = lib.mkOption { + description = "Encryption key for the backups."; + type = lib.types.path; + }; + + sourceDirectories = lib.mkOption { + description = "Source directories."; + type = lib.types.nonEmptyListOf lib.types.str; + }; + + excludePatterns = lib.mkOption { + description = "Exclude patterns."; + type = lib.types.listOf lib.types.str; + default = []; + }; + + repositories = lib.mkOption { + description = "Repositories to back this instance to."; + type = lib.types.nonEmptyListOf (lib.types.submodule { + options = { + path = lib.mkOption { + type = lib.types.str; + description = "Repository location"; + }; + + extraSecrets = lib.mkOption { + type = lib.types.attrsOf shblib.secretFileType; + default = {}; + description = '' + Extra secrets needed to access the repository where the backups will be stored. + + See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example. + ''; + example = lib.literalExpression '' + { + AWS_ACCESS_KEY_ID = ; + AWS_SECRET_ACCESS_KEY = ; + } + ''; + }; + + timerConfig = lib.mkOption { + type = lib.types.attrsOf utils.systemdUtils.unitOptions.unitOption; + default = { + OnCalendar = "daily"; + Persistent = true; + }; + description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.''; + example = { + OnCalendar = "00:05"; + RandomizedDelaySec = "5h"; + Persistent = true; + }; + }; + }; + }); + }; + + retention = lib.mkOption { + description = "For how long to keep backup files."; + type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]); + default = { + keep_within = "1d"; + keep_hourly = 24; + keep_daily = 7; + keep_weekly = 4; + keep_monthly = 6; + }; + }; + + hooks = lib.mkOption { + description = "Hooks to run before or after the backup."; + default = {}; + type = lib.types.submodule { + options = { + before_backup = lib.mkOption { + description = "Hooks to run before backup"; + type = lib.types.listOf lib.types.str; + default = []; + }; + + after_backup = lib.mkOption { + description = "Hooks to run after backup"; + type = lib.types.listOf lib.types.str; + default = []; + }; + }; + }; + }; + }; + + repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (lib.strings.removePrefix "/" name); +in +{ + options.shb.restic = { + user = lib.mkOption { + description = "Unix user doing the backups."; + type = lib.types.str; + default = "backup"; + }; + + group = lib.mkOption { + description = "Unix group doing the backups."; + type = lib.types.str; + default = "backup"; + }; + + instances = lib.mkOption { + description = "Each instance is a backup setting"; + default = {}; + type = lib.types.attrsOf (lib.types.submodule { + options = instanceOptions; + }); + }; + + # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23 + performance = lib.mkOption { + description = "Reduce performance impact of backup jobs."; + default = {}; + type = lib.types.submodule { + options = { + niceness = lib.mkOption { + type = lib.types.ints.between (-20) 19; + description = "nice priority adjustment, defaults to 15 for ~20% CPU time of normal-priority process"; + default = 15; + }; + ioSchedulingClass = lib.mkOption { + type = lib.types.enum [ "idle" "best-effort" "realtime" ]; + description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands."; + default = "best-effort"; + }; + ioPriority = lib.mkOption { + type = lib.types.nullOr (lib.types.ints.between 0 7); + description = "ionice priority, defaults to 7 for lowest priority IO. Only used for `restic backup`, `restic forget` and `restic check` commands."; + default = 7; + }; + }; + }; + }; + }; + + config = lib.mkIf (cfg.instances != {}) ( + let + enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances; + in lib.mkMerge [ + { + users.users = { + ${cfg.user} = { + name = cfg.user; + group = cfg.group; + home = lib.mkForce "/var/lib/${cfg.user}"; + createHome = true; + isSystemUser = true; + extraGroups = [ "keys" ]; + }; + }; + users.groups = { + ${cfg.group} = { + name = cfg.group; + }; + }; + } + { + environment.systemPackages = lib.optionals (enabledInstances != {}) [ pkgs.restic ]; + + services.restic.backups = + let + mkRepositorySettings = name: instance: repository: { + "${name}_${repoSlugName repository.path}" = { + inherit (cfg) user; + repository = repository.path; + + paths = instance.sourceDirectories; + + passwordFile = toString instance.passphraseFile; + + initialize = true; + + inherit (repository) timerConfig; + + pruneOpts = lib.mapAttrsToList (name: value: + "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}" + ) instance.retention; + + backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup; + + backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup; + } // lib.attrsets.optionalAttrs (repository.extraSecrets != {}) { + environmentFile = shblib.replaceSecrets { + userConfig = repository.extraSecrets; + resultPath = "/var/lib/backup/${name}"; + generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} v); + }; + } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) { + exclude = instance.excludePatterns; + }; + }; + + mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; + in + lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances)); + + systemd.services = + let + mkRepositorySettings = name: instance: repository: { + "restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = { + Nice = cfg.performance.niceness; + IOSchedulingClass = cfg.performance.ioSchedulingClass; + IOSchedulingPriority = cfg.performance.ioPriority; + }; + }; + mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; + in + lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances)); + } + { + environment.systemPackages = let + mkResticBinary = name: instance: repository: pkgs.writeShellScriptBin "restic-${name}_${repoSlugName repository.path}" '' + export RESTIC_PASSWORD_FILE=${instance.passphraseFile} + export RESTIC_REPOSITORY=${repository.path} + ${pkgs.restic}/bin/restic $@ + ''; + mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.repositories; + in + lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances); + } + ]); +} diff --git a/modules/blocks/backup/docs/default.md b/modules/blocks/restic/docs/default.md similarity index 60% rename from modules/blocks/backup/docs/default.md rename to modules/blocks/restic/docs/default.md index c431b4f..7e09165 100644 --- a/modules/blocks/backup/docs/default.md +++ b/modules/blocks/restic/docs/default.md @@ -1,33 +1,32 @@ -# Backup Block {#blocks-backup} +# Restic Block {#blocks-restic} -Defined in [`/modules/blocks/backup.nix`](@REPO@/modules/blocks/backup.nix). +Defined in [`/modules/blocks/restic.nix`](@REPO@/modules/blocks/restic.nix). -This block sets up backup jobs for Self Host Blocks. +This block sets up a backup job using [Restic][restic]. -## Features {#blocks-backup-features} -Two implementations for this block are provided: -- [Restic](https://restic.net/) -- [Borgmatic](https://torsion.org/borgmatic/) +[restic]: https://restic.net/ -No integration tests are provided yet. +## Contract {#blocks-restic-features} + +This block implements the [backup](contracts-backup.html) contract. + +Integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix). ## Usage {#blocks-backup-usage} ### One folder backed up to mounted hard drives {#blocks-backup-usage-one} -The following snippet shows how to configure backup of 1 folder using the Restic implementation to 1 -repository. +The following snippet shows how to configure +the backup of 1 folder to 1 repository. Assumptions: - 1 hard drive pool is used for backup and is mounted on `/srv/pool1`. ```nix -shb.backup.instances.myfolder = { +shb.restic.instances.myfolder = { enable = true; - backend = "restic"; - - keySopsFile = ./secrets.yaml; + passphraseFile = ""; repositories = [{ path = "/srv/pool1/backups/myfolder"; @@ -56,36 +55,14 @@ shb.backup.instances.myfolder = { }; ``` -The referenced Sops file must follow this structure: +To be secure, the `passphraseFile` must contain a secret that is deployed out of band, otherwise it will be world-readable in the nix store. +To achieve that, I recommend [sops](usage.html#usage-secrets) although other methods work great too. -```yaml -restic: - passphrases: - myfolder: -``` - -To generate a secret, use: `nix run nixpkgs#openssl -- rand -hex 64`. - -With the borgmatic implementation, the structure should be: - -```yaml -borgmatic: - keys: - myfolder: | - BORG_KEY - passphrases: - myfolder: -``` - -You can have both borgmatic and restic implementations working at the same time. - -### One folder backed up to S3 {#blocks-backup-usage-remote} - -> This is only supported by the Restic implementation. +### One folder backed up to S3 {#blocks-restic-usage-remote} Here we will only highlight the differences with the previous configuration. -This assumes you have access to such a remote S3 store, for example by using Backblaze. +This assumes you have access to such a remote S3 store, for example by using [Backblaze](https://www.backblaze.com/). ```diff shb.backup.instances.myfolder = { @@ -97,31 +74,19 @@ This assumes you have access to such a remote S3 store, for example by using Bac OnCalendar = "00:00:00"; RandomizedDelaySec = "3h"; }; + ++ extraSecrets = { ++ AWS_ACCESS_KEY_ID=""; ++ AWS_SECRET_ACCESS_KEY=""; ++ }; }]; - - -+ environmentFile = true; # Needed for s3 } ``` -The Sops file has a new required field: +### Multiple directories to multiple destinations {#blocks-restic-usage-multiple} -```yaml - - restic: - passphrases: - myfolder: -+ environmentfiles: -+ myfolder: |- -+ AWS_ACCESS_KEY_ID= -+ AWS_SECRET_ACCESS_KEY= -``` - -### Multiple folder to multiple destinations {#blocks-backup-usage-multiple} - -The following snippet shows how to configure backup of any number of folders using the Restic -implementation to 3 repositories, each happening at different times to avoid contending for I/O -time. +The following snippet shows how to configure backup of any number of folders to 3 repositories, +each happening at different times to avoid I/O contention. We will also make sure to be able to re-use as much as the configuration as possible. @@ -129,7 +94,7 @@ A few assumptions: - 2 hard drive pools used for backup are mounted respectively on `/srv/pool1` and `/srv/pool2`. - You have a backblaze account. -First, let's define a variable to hold all our repositories you want to back up to: +First, let's define a variable to hold all the repositories we want to back up to: ```nix repos = [ @@ -209,19 +174,38 @@ below) is the former splits the backups into sub-folders on the repositories. shb.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"]; ``` -## Demo {#blocks-backup-demo} +## Demo {#blocks-restic-demo} [WIP] -## Monitoring {#blocks-backup-monitoring} +## Monitoring {#blocks-restic-monitoring} [WIP] -## Maintenance {#blocks-backup-maintenance} +## Maintenance {#blocks-restic-maintenance} -[WIP] +One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets. -## Options Reference {#blocks-backup-options} +In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`: + +```bash +restic-myfolder1_srv_pool1_backups +restic-myfolder1_srv_pool2_backups +restic-myfolder1_s3_s3.us-west-000.backblazeb2.com_backups +restic-myfolder2_srv_pool1_backups +restic-myfolder2_srv_pool2_backups +restic-myfolder2_s3_s3.us-west-000.backblazeb2.com_backups +``` + +Discovering those is easy thanks to tab-completion. + +One can then restore a backup with: + +```bash +restic-myfolder1_srv_pool1_backups restore latest -t / +``` + +## Options Reference {#blocks-restic-options} ```{=include=} options id-prefix: blocks-backup-options- diff --git a/modules/contracts/backup.nix b/modules/contracts/backup.nix new file mode 100644 index 0000000..0f983d4 --- /dev/null +++ b/modules/contracts/backup.nix @@ -0,0 +1,61 @@ +{ lib, ... }: +lib.types.submodule { + freeformType = lib.types.anything; + + options = { + user = lib.mkOption { + description = "Unix user doing the backups."; + type = lib.types.str; + default = "backup"; + }; + + group = lib.mkOption { + description = "Unix group doing the backups."; + type = lib.types.str; + default = "backup"; + }; + + sourceDirectories = lib.mkOption { + description = "Directories to backup."; + type = lib.types.nonEmptyListOf lib.types.str; + }; + + excludePatterns = lib.mkOption { + description = "Patterns to exclude."; + type = lib.types.listOf lib.types.str; + default = []; + }; + + retention = lib.mkOption { + description = "Backup files retention."; + type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]); + default = { + keep_within = "1d"; + keep_hourly = 24; + keep_daily = 7; + keep_weekly = 4; + keep_monthly = 6; + }; + }; + + hooks = lib.mkOption { + description = "Hooks to run around the backup."; + default = {}; + type = lib.types.submodule { + options = { + before_backup = lib.mkOption { + description = "Hooks to run before backup"; + type = lib.types.listOf lib.types.str; + default = []; + }; + + after_backup = lib.mkOption { + description = "Hooks to run after backup"; + type = lib.types.listOf lib.types.str; + default = []; + }; + }; + }; + }; + }; +} diff --git a/modules/contracts/backup/docs/default.md b/modules/contracts/backup/docs/default.md new file mode 100644 index 0000000..4e8d790 --- /dev/null +++ b/modules/contracts/backup/docs/default.md @@ -0,0 +1,56 @@ +# Backup Contract {#backup-contract} + +This NixOS contract represents a backup job +that will backup one or more files or directories +at a regular schedule. + +## Contract Reference {#backup-contract-options} + +These are all the options that are expected to exist for this contract to be respected. + +```{=include=} options +id-prefix: contracts-backup-options- +list-id: selfhostblocks-options +source: @OPTIONS_JSON@ +``` + +## Usage {#backup-contract-usage} + +A service that can be backed up will provide a `backup` option, like for the [Vaultwarden service][vaultwarden-service-backup]. +What this option defines is an implementation detail of that service +but it will at least define what directories to backup +and possibly hooks to run before or after the backup job runs. + +[vaultwarden-service-backup]: services-vaultwarden.html#services-vaultwarden-options-shb.vaultwarden.backup + +```nix +shb..backup +``` + +Let's assume a module implementing this contract is available under the `shb.` variable. +Then, to actually backup the service, one would write: + +```nix +shb..instances."" = shb..backup // { + enable = true; + + // Options specific to backup_impl +}; +``` + +Then, for extra caution, a second backup could be made using another module `shb.`: + +```nix +shb..instances."" = shb..backup // { + enable = true; + + // Options specific to backup_impl_2 +}; +``` + +## Provided Implementations {#backup-contract-impl} + +One implementation is provided out of the box: +- [Restic block](blocks-restic.html). + +A second one based on `borgbackup` is in progress. diff --git a/modules/contracts/backup/dummyModule.nix b/modules/contracts/backup/dummyModule.nix new file mode 100644 index 0000000..8a7d3a8 --- /dev/null +++ b/modules/contracts/backup/dummyModule.nix @@ -0,0 +1,10 @@ +{ pkgs, lib, ... }: +let + contracts = pkgs.callPackage ../. {}; +in +{ + options.shb.contracts.backup = lib.mkOption { + description = "Contract for backups."; + type = contracts.backup; + }; +} diff --git a/modules/contracts/default.nix b/modules/contracts/default.nix index 54a06f6..e748008 100644 --- a/modules/contracts/default.nix +++ b/modules/contracts/default.nix @@ -1,5 +1,6 @@ { lib }: { + backup = import ./backup.nix { inherit lib; }; mount = import ./mount.nix { inherit lib; }; ssl = import ./ssl.nix { inherit lib; }; } diff --git a/modules/services/vaultwarden.nix b/modules/services/vaultwarden.nix index ddfdfa0..21ee588 100644 --- a/modules/services/vaultwarden.nix +++ b/modules/services/vaultwarden.nix @@ -114,10 +114,26 @@ in default = { path = dataFolder; }; }; - backupConfig = lib.mkOption { - type = lib.types.nullOr lib.types.anything; - description = "Backup configuration of Vaultwarden."; - default = null; + backup = lib.mkOption { + type = contracts.backup; + description = '' + Backup configuration. This is an output option. + + Use it to initialize a block implementing the "backup" contract. + For example, with the restic block: + + ``` + shb.restic.instances."vaultwarden" = { + poolName = "root"; + } // config.shb.vaultwarden.backup; + ``` + ''; + readOnly = true; + default = { + sourceDirectories = [ + dataFolder + ]; + }; }; debug = lib.mkOption { @@ -217,14 +233,6 @@ in members = [ "backup" ]; }; - shb.backup.instances.vaultwarden = lib.mkIf (cfg.backupConfig != null) ( - cfg.backupConfig // - { - sourceDirectories = [ - config.services.vaultwarden.config.DATA_FOLDER - ]; - }); - # TODO: make this work. # It does not work because it leads to infinite recursion. # ${cfg.mount}.path = dataFolder; diff --git a/modules/services/vaultwarden/docs/default.md b/modules/services/vaultwarden/docs/default.md index 5886f7c..af1a6d1 100644 --- a/modules/services/vaultwarden/docs/default.md +++ b/modules/services/vaultwarden/docs/default.md @@ -91,3 +91,25 @@ Integration with the ZFS block allows to automatically create the relevant datas shb.zfs.datasets."vaultwarden" = config.shb.vaultwarden.mount; shb.zfs.datasets."postgresql".path = "/var/lib/postgresql"; ``` + +## Maintenance {#services-vaultwarden-maintenance} + +No command-line tool is provided to administer Vaultwarden. + +Instead, the admin section can be found at the `/admin` endpoint. + +## Debug {#services-backup-debug} + +In case of an issue, check the logs of the `vaultwarden.service` systemd service. + +Enable verbose logging by setting the `shb.vaultwarden.debug` boolean to `true`. + +Access the database with `sudo -u vaultwarden psql`. + +## Options Reference {#services-vaultwarden-options} + +```{=include=} options +id-prefix: services-vaultwarden-options- +list-id: selfhostblocks-vaultwarden-options +source: @OPTIONS_JSON@ +``` diff --git a/test/blocks/restic.nix b/test/blocks/restic.nix new file mode 100644 index 0000000..12863c5 --- /dev/null +++ b/test/blocks/restic.nix @@ -0,0 +1,156 @@ +{ pkgs, lib, ... }: +let + pkgs' = pkgs; + + testLib = pkgs.callPackage ../common.nix {}; + + base = testLib.base [ + ../../modules/blocks/restic.nix + ]; +in +{ + backupAndRestore = pkgs.testers.runNixOSTest { + name = "restic_backupAndRestore"; + + nodes.machine = { + imports = ( testLib.baseImports pkgs' ) ++ [ + ../../modules/blocks/restic.nix + ]; + + shb.restic = { + user = "root"; + group = "root"; + }; + shb.restic.instances."testinstance" = { + enable = true; + + passphraseFile = pkgs.writeText "passphrase" "PassPhrase"; + + sourceDirectories = [ + "/opt/files/A" + "/opt/files/B" + ]; + + repositories = [ + { + path = "/opt/repos/A"; + timerConfig = { + OnCalendar = "00:00:00"; + RandomizedDelaySec = "5h"; + }; + } + { + path = "/opt/repos/B"; + timerConfig = { + OnCalendar = "00:00:00"; + RandomizedDelaySec = "5h"; + }; + } + ]; + }; + }; + + extraPythonPackages = p: [ p.dictdiffer ]; + skipTypeCheck = true; + + testScript = { nodes, ... }: let + instanceCfg = nodes.machine.shb.restic.instances."testinstance"; + in '' + from dictdiffer import diff + + def list_files(dir): + files_and_content = {} + + files = machine.succeed(f""" + find {dir} -type f + """).split("\n")[:-1] + + for f in files: + content = machine.succeed(f""" + cat {f} + """).strip() + files_and_content[f] = content + + return files_and_content + + def assert_files(dir, files): + result = list(diff(list_files(dir), files)) + if len(result) > 0: + raise Exception("Unexpected files:", result) + + with subtest("Create initial content"): + machine.succeed(""" + mkdir -p /opt/files/A + mkdir -p /opt/files/B + mkdir -p /opt/repos/A + mkdir -p /opt/repos/B + + echo repoA_fileA_1 > /opt/files/A/fileA + echo repoA_fileB_1 > /opt/files/A/fileB + echo repoB_fileA_1 > /opt/files/B/fileA + echo repoB_fileB_1 > /opt/files/B/fileB + + # chown :backup -R /opt/files + """) + + assert_files("/opt/files", { + '/opt/files/B/fileA': 'repoB_fileA_1', + '/opt/files/B/fileB': 'repoB_fileB_1', + '/opt/files/A/fileA': 'repoA_fileA_1', + '/opt/files/A/fileB': 'repoA_fileB_1', + }) + + with subtest("First backup in repo A"): + machine.succeed("systemctl start restic-backups-testinstance_opt_repos_A") + + with subtest("New content"): + machine.succeed(""" + echo repoA_fileA_2 > /opt/files/A/fileA + echo repoA_fileB_2 > /opt/files/A/fileB + echo repoB_fileA_2 > /opt/files/B/fileA + echo repoB_fileB_2 > /opt/files/B/fileB + """) + + assert_files("/opt/files", { + '/opt/files/B/fileA': 'repoB_fileA_2', + '/opt/files/B/fileB': 'repoB_fileB_2', + '/opt/files/A/fileA': 'repoA_fileA_2', + '/opt/files/A/fileB': 'repoA_fileB_2', + }) + + with subtest("Second backup in repo B"): + machine.succeed("systemctl start restic-backups-testinstance_opt_repos_B") + + with subtest("Delete content"): + machine.succeed(""" + rm -r /opt/files/A /opt/files/B + """) + + assert_files("/opt/files", {}) + + with subtest("Restore initial content from repo A"): + machine.succeed(""" + restic-testinstance_opt_repos_A restore latest -t / + """) + + assert_files("/opt/files", { + '/opt/files/B/fileA': 'repoB_fileA_1', + '/opt/files/B/fileB': 'repoB_fileB_1', + '/opt/files/A/fileA': 'repoA_fileA_1', + '/opt/files/A/fileB': 'repoA_fileB_1', + }) + + with subtest("Restore initial content from repo B"): + machine.succeed(""" + restic-testinstance_opt_repos_B restore latest -t / + """) + + assert_files("/opt/files", { + '/opt/files/B/fileA': 'repoB_fileA_2', + '/opt/files/B/fileB': 'repoB_fileB_2', + '/opt/files/A/fileA': 'repoA_fileA_2', + '/opt/files/A/fileB': 'repoA_fileB_2', + }) + ''; + }; +} diff --git a/test/common.nix b/test/common.nix index abeef15..88a1322 100644 --- a/test/common.nix +++ b/test/common.nix @@ -1,6 +1,12 @@ { lib, }: +let + baseImports = pkgs: [ + (pkgs.path + "/nixos/modules/profiles/headless.nix") + (pkgs.path + "/nixos/modules/profiles/qemu-guest.nix") + ]; +in { accessScript = { subdomain @@ -92,21 +98,24 @@ ${indent 4 script} ''); + inherit baseImports; + base = pkgs: additionalModules: { - imports = [ - (pkgs.path + "/nixos/modules/profiles/headless.nix") - (pkgs.path + "/nixos/modules/profiles/qemu-guest.nix") - # TODO: replace this option by the backup contract - { - options = { - shb.backup = lib.mkOption { type = lib.types.anything; }; - }; - } - # TODO: replace postgresql.nix and authelia.nix by the sso contract - ../modules/blocks/postgresql.nix - ../modules/blocks/authelia.nix - ../modules/blocks/nginx.nix - ] ++ additionalModules; + imports = + ( baseImports pkgs ) + ++ [ + # TODO: replace this option by the backup contract + { + options = { + shb.backup = lib.mkOption { type = lib.types.anything; }; + }; + } + # TODO: replace postgresql.nix and authelia.nix by the sso contract + ../modules/blocks/postgresql.nix + ../modules/blocks/authelia.nix + ../modules/blocks/nginx.nix + ] + ++ additionalModules; # Nginx port. networking.firewall.allowedTCPPorts = [ 80 443 ];