add backup contract

docs/blocks.md

@@ -34,8 +34,8 @@ Not all blocks are yet documented. You can find all available blocks [in the rep
 modules/blocks/ssl/docs/default.md
 ```
 
-```{=include=} chapters html:into-file=//blocks-backup.html
+```{=include=} chapters html:into-file=//blocks-restic.html
-modules/blocks/backup/docs/default.md
+modules/blocks/restic/docs/default.md
 ```
 
 ```{=include=} chapters html:into-file=//blocks-monitoring.html

docs/contracts.md

@@ -19,12 +19,19 @@ as possible, reducing the quite thick layer that it is now.
 
 Provided contracts are:
 
-- [SSL generator contract](contracts-ssl.html) to generate SSL certificates. Two implementations are provided: self-signed and Let's Encrypt.
+- [SSL generator contract](contracts-ssl.html) to generate SSL certificates.
+  Two implementations are provided: self-signed and Let's Encrypt.
+- [Backup contract](contracts-backup.html) to backup directories.
+  This contract allows to backup multiple times the same directories for extra protection.
 
 ```{=include=} chapters html:into-file=//contracts-ssl.html
 modules/contracts/ssl/docs/default.md
 ```
 
+```{=include=} chapters html:into-file=//contracts-backup.html
+modules/contracts/backup/docs/default.md
+```
+
 ## Why do we need this new concept? {#contracts-why}
 
 Currently in nixpkgs, every module needing access to a shared resource must implement the logic

docs/default.nix

@@ -67,7 +67,9 @@ let
   };
 
   optionsDocs = buildOptionsDocs {
-    modules = allModules ++ [ scrubbedModule ];
+    modules = allModules ++ [
+      scrubbedModule
+    ];
     variablelistId = "selfhostblocks-options";
     includeModuleSystemOptions = false;
   };
@@ -134,10 +136,10 @@ in stdenv.mkDerivation {
         '@OPTIONS_JSON@' \
         ${individualModuleOptionsDocs ../modules/blocks/ssl.nix}/share/doc/nixos/options.json
 
-    substituteInPlace ./modules/blocks/backup/docs/default.md \
+    substituteInPlace ./modules/blocks/restic/docs/default.md \
       --replace \
         '@OPTIONS_JSON@' \
-        ${individualModuleOptionsDocs ../modules/blocks/backup.nix}/share/doc/nixos/options.json
+        ${individualModuleOptionsDocs ../modules/blocks/restic.nix}/share/doc/nixos/options.json
 
     substituteInPlace ./modules/services/nextcloud-server/docs/default.md \
       --replace \
@@ -149,6 +151,11 @@ in stdenv.mkDerivation {
         '@OPTIONS_JSON@' \
        ${individualModuleOptionsDocs ../modules/services/vaultwarden.nix}/share/doc/nixos/options.json
 
+    substituteInPlace ./modules/contracts/backup/docs/default.md \
+      --replace \
+        '@OPTIONS_JSON@' \
+       ${individualModuleOptionsDocs ../modules/contracts/backup/dummyModule.nix}/share/doc/nixos/options.json
+
     substituteInPlace ./modules/contracts/ssl/docs/default.md \
       --replace \
         '@OPTIONS_JSON@' \

flake.lock

@@ -49,38 +49,6 @@
         "type": "github"
       }
     },
-    "nixpkgs-stable": {
-      "locked": {
-        "lastModified": 1716655032,
-        "narHash": "sha256-kQ25DAiCGigsNR/Quxm3v+JGXAEXZ8I7RAF4U94bGzE=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "59a450646ec8ee0397f5fa54a08573e8240eb91f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "release-23.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1716651315,
-        "narHash": "sha256-iMgzIeedMqf30TXZ439zW3Yvng1Xm9QTGO+ZwG1IWSw=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "c5187508b11177ef4278edf19616f44f21cc8c69",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "nmdsrc": {
       "flake": false,
       "locked": {
@@ -102,27 +70,7 @@
         "flake-utils": "flake-utils",
         "nix-flake-tests": "nix-flake-tests",
         "nixpkgs": "nixpkgs",
-        "nmdsrc": "nmdsrc",
+        "nmdsrc": "nmdsrc"
-        "sops-nix": "sops-nix"
-      }
-    },
-    "sops-nix": {
-      "inputs": {
-        "nixpkgs": "nixpkgs_2",
-        "nixpkgs-stable": "nixpkgs-stable"
-      },
-      "locked": {
-        "lastModified": 1716692524,
-        "narHash": "sha256-sALodaA7Zkp/JD6ehgwc0UCBrSBfB4cX66uFGTsqeFU=",
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "rev": "962797a8d7f15ed7033031731d0bb77244839960",
-        "type": "github"
-      },
-      "original": {
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "type": "github"
       }
     },
     "systems": {

flake.nix

@@ -36,12 +36,12 @@
 
       allModules = [
         modules/blocks/authelia.nix
-        modules/blocks/backup.nix
         modules/blocks/davfs.nix
         modules/blocks/ldap.nix
         modules/blocks/monitoring.nix
         modules/blocks/nginx.nix
         modules/blocks/postgresql.nix
+        modules/blocks/restic.nix
         modules/blocks/ssl.nix
         modules/blocks/tinyproxy.nix
         modules/blocks/vpn.nix
@@ -60,6 +60,7 @@
 
       # Only used for documentation.
       contractDummyModules = [
+        modules/contracts/backup/dummyModule.nix
         modules/contracts/ssl/dummyModule.nix
       ];
     in
@@ -133,6 +134,7 @@
           // (vm_test "ldap" ./test/blocks/ldap.nix)
           // (vm_test "lib" ./test/blocks/lib.nix)
           // (vm_test "postgresql" ./test/blocks/postgresql.nix)
+          // (vm_test "restic" ./test/blocks/restic.nix)
           // (vm_test "ssl" ./test/blocks/ssl.nix)
           );
       }

modules/blocks/borgbackup.nix

@@ -1,30 +1,32 @@
 { config, pkgs, lib, utils, ... }:
 
 let
-  cfg = config.shb.backup;
+  cfg = config.shb.borgbackup;
 
   instanceOptions = {
-    enable = lib.mkEnableOption "shb backup instance";
+    enable = lib.mkEnableOption "shb borgbackup";
-
-    backend = lib.mkOption {
-      description = "What program to use to make the backups.";
-      type = lib.types.enum [ "borgmatic" "restic" ];
-      example = "borgmatic";
-    };
 
     keySopsFile = lib.mkOption {
-      description = "Sops file that holds this instance's Borgmatic repository key and passphrase.";
+      description = "Sops file that holds this instance's repository key and passphrase.";
       type = lib.types.path;
       example = "secrets/backup.yaml";
     };
 
+    encryptionKeyFile = lib.mkOption {
+      description = "Encryption key for the backup.";
+      type = lib.types.path;
+    };
+
+    encryption_passcommand = "cat /run/secrets/borgmatic/passphrases/${if isNull instance.secretName then name else instance.secretName}";
+    borg_keys_directory = "/run/secrets/borgmatic/keys";
+
     sourceDirectories = lib.mkOption {
-      description = "Borgmatic source directories.";
+      description = "Source directories.";
       type = lib.types.nonEmptyListOf lib.types.str;
     };
 
     excludePatterns = lib.mkOption {
-      description = "Borgmatic exclude patterns.";
+      description = "Exclude patterns.";
       type = lib.types.listOf lib.types.str;
       default = [];
     };
@@ -74,7 +76,7 @@ let
     };
 
     consistency = lib.mkOption {
-      description = "Consistency frequency options. Only applicable for borgmatic";
+      description = "Consistency frequency options.";
       type = lib.types.attrsOf lib.types.nonEmptyStr;
       default = {};
       example = {
@@ -84,7 +86,7 @@ let
     };
 
     hooks = lib.mkOption {
-      description = "Borgmatic hooks.";
+      description = "Hooks to run before or after the backup.";
       default = {};
       type = lib.types.submodule {
         options = {
@@ -115,14 +117,7 @@ let
 
 in
 {
-  options.shb.backup = {
+  options.shb.borgbackup = {
-    onlyOnAC = lib.mkOption {
-      description = "Run backups only if AC power is plugged in.";
-      default = true;
-      example = false;
-      type = lib.types.bool;
-    };
-
     user = lib.mkOption {
       description = "Unix user doing the backups.";
       type = lib.types.str;
@@ -163,12 +158,12 @@ in
           };
           ioSchedulingClass = lib.mkOption {
             type = lib.types.enum [ "idle" "best-effort" "realtime" ];
-            description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
+            description = "ionice scheduling class, defaults to best-effort IO.";
             default = "best-effort";
           };
           ioPriority = lib.mkOption {
             type = lib.types.nullOr (lib.types.ints.between 0 7);
-            description = "ionice priority, defaults to 7 for lowest priority IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
+            description = "ionice priority, defaults to 7 for lowest priority IO.";
             default = 7;
           };
         };
@@ -179,8 +174,6 @@ in
   config = lib.mkIf (cfg.instances != {}) (
     let
       enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
-      borgmaticInstances = lib.attrsets.filterAttrs (k: i: i.backend == "borgmatic") enabledInstances;
-      resticInstances = lib.attrsets.filterAttrs (k: i: i.backend == "restic") enabledInstances;
     in lib.mkMerge [
       # Secrets configuration
       {
@@ -234,13 +227,13 @@ in
       }
       # Borgmatic configuration
       {
-        systemd.timers.borgmatic = lib.mkIf (borgmaticInstances != {}) {
+        systemd.timers.borgmatic = lib.mkIf (enabledInstances != {}) {
           timerConfig = {
             OnCalendar = "hourly";
           };
         };
 
-        systemd.services.borgmatic = lib.mkIf (borgmaticInstances != {}) {
+        systemd.services.borgmatic = lib.mkIf (enabledInstances != {}) {
           serviceConfig = {
             User = cfg.user;
             Group = cfg.group;
@@ -252,10 +245,10 @@ in
           };
         };
 
-        systemd.packages = lib.mkIf (borgmaticInstances != {}) [ pkgs.borgmatic ];
+        systemd.packages = lib.mkIf (enabledInstances != {}) [ pkgs.borgmatic ];
         environment.systemPackages = (
           lib.optionals cfg.borgServer [ pkgs.borgbackup ]
-          ++ lib.optionals (borgmaticInstances != {}) [ pkgs.borgbackup pkgs.borgmatic ]
+          ++ lib.optionals (enabledInstances != {}) [ pkgs.borgbackup pkgs.borgmatic ]
         );
 
         environment.etc =
@@ -272,7 +265,7 @@ in
                   });
 
                 storage = {
-                  encryption_passcommand = "cat /run/secrets/borgmatic/passphrases/${if isNull instance.secretName then name else instance.secretName}";
+                  encryption_passcommand = "cat ${instance.encryptionKeyFile}";
                   borg_keys_directory = "/run/secrets/borgmatic/keys";
                 };
 
@@ -296,57 +289,7 @@ in
               };
             };
           in
-            lib.mkMerge (lib.attrsets.mapAttrsToList mkSettings borgmaticInstances);
+            lib.mkMerge (lib.attrsets.mapAttrsToList mkSettings enabledInstances);
-      }
-      # Restic configuration
-      {
-        environment.systemPackages = lib.optionals (resticInstances != {}) [ pkgs.restic ];
-
-        services.restic.backups =
-          let
-            mkRepositorySettings = name: instance: repository: {
-              "${name}_${repoSlugName repository.path}" = {
-                inherit (cfg) user;
-                repository = repository.path;
-
-                paths = instance.sourceDirectories;
-
-                passwordFile = "/run/secrets/${instance.backend}/passphrases/${name}";
-
-                initialize = true;
-
-                inherit (repository) timerConfig;
-
-                pruneOpts = lib.mapAttrsToList (name: value:
-                  "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
-                ) instance.retention;
-
-                backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
-
-                backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
-              } // lib.attrsets.optionalAttrs (instance.environmentFile) {
-                environmentFile = "/run/secrets/${instance.backend}/environmentfiles/${name}";
-              } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
-                exclude = instance.excludePatterns;
-              };
-            };
-
-            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
-          in
-            lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings resticInstances));
-
-        systemd.services =
-          let
-            mkRepositorySettings = name: instance: repository: {
-              "restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = {
-                Nice = cfg.performance.niceness;
-                IOSchedulingClass = cfg.performance.ioSchedulingClass;
-                IOSchedulingPriority = cfg.performance.ioPriority;
-              };
-            };
-            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
-          in
-            lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings resticInstances));
       }
     ]);
 }

modules/blocks/restic.nix

@@ -0,0 +1,238 @@
+{ config, pkgs, lib, utils, ... }:
+
+let
+  cfg = config.shb.restic;
+
+  shblib = pkgs.callPackage ../../lib {};
+
+  instanceOptions = {
+    enable = lib.mkEnableOption "shb restic";
+
+    passphraseFile = lib.mkOption {
+      description = "Encryption key for the backups.";
+      type = lib.types.path;
+    };
+
+    sourceDirectories = lib.mkOption {
+      description = "Source directories.";
+      type = lib.types.nonEmptyListOf lib.types.str;
+    };
+
+    excludePatterns = lib.mkOption {
+      description = "Exclude patterns.";
+      type = lib.types.listOf lib.types.str;
+      default = [];
+    };
+
+    repositories = lib.mkOption {
+      description = "Repositories to back this instance to.";
+      type = lib.types.nonEmptyListOf (lib.types.submodule {
+        options = {
+          path = lib.mkOption {
+            type = lib.types.str;
+            description = "Repository location";
+          };
+
+          extraSecrets = lib.mkOption {
+            type = lib.types.attrsOf shblib.secretFileType;
+            default = {};
+            description = ''
+              Extra secrets needed to access the repository where the backups will be stored.
+
+              See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example.
+              '';
+            example = lib.literalExpression ''
+              {
+                AWS_ACCESS_KEY_ID = <path/to/secret>;
+                AWS_SECRET_ACCESS_KEY = <path/to/secret>;
+              }
+              '';
+          };
+
+          timerConfig = lib.mkOption {
+            type = lib.types.attrsOf utils.systemdUtils.unitOptions.unitOption;
+            default = {
+              OnCalendar = "daily";
+              Persistent = true;
+            };
+            description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
+            example = {
+              OnCalendar = "00:05";
+              RandomizedDelaySec = "5h";
+              Persistent = true;
+            };
+          };
+        };
+      });
+    };
+
+    retention = lib.mkOption {
+      description = "For how long to keep backup files.";
+      type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]);
+      default = {
+        keep_within = "1d";
+        keep_hourly = 24;
+        keep_daily = 7;
+        keep_weekly = 4;
+        keep_monthly = 6;
+      };
+    };
+
+    hooks = lib.mkOption {
+      description = "Hooks to run before or after the backup.";
+      default = {};
+      type = lib.types.submodule {
+        options = {
+          before_backup = lib.mkOption {
+            description = "Hooks to run before backup";
+            type = lib.types.listOf lib.types.str;
+            default = [];
+          };
+
+          after_backup = lib.mkOption {
+            description = "Hooks to run after backup";
+            type = lib.types.listOf lib.types.str;
+            default = [];
+          };
+        };
+      };
+    };
+  };
+
+  repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (lib.strings.removePrefix "/" name);
+in
+{
+  options.shb.restic = {
+    user = lib.mkOption {
+      description = "Unix user doing the backups.";
+      type = lib.types.str;
+      default = "backup";
+    };
+
+    group = lib.mkOption {
+      description = "Unix group doing the backups.";
+      type = lib.types.str;
+      default = "backup";
+    };
+
+    instances = lib.mkOption {
+      description = "Each instance is a backup setting";
+      default = {};
+      type = lib.types.attrsOf (lib.types.submodule {
+        options = instanceOptions;
+      });
+    };
+
+    # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23
+    performance = lib.mkOption {
+      description = "Reduce performance impact of backup jobs.";
+      default = {};
+      type = lib.types.submodule {
+        options = {
+          niceness = lib.mkOption {
+            type = lib.types.ints.between (-20) 19;
+            description = "nice priority adjustment, defaults to 15 for ~20% CPU time of normal-priority process";
+            default = 15;
+          };
+          ioSchedulingClass = lib.mkOption {
+            type = lib.types.enum [ "idle" "best-effort" "realtime" ];
+            description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
+            default = "best-effort";
+          };
+          ioPriority = lib.mkOption {
+            type = lib.types.nullOr (lib.types.ints.between 0 7);
+            description = "ionice priority, defaults to 7 for lowest priority IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
+            default = 7;
+          };
+        };
+      };
+    };
+  };
+
+  config = lib.mkIf (cfg.instances != {}) (
+    let
+      enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
+    in lib.mkMerge [
+      {
+        users.users = {
+          ${cfg.user} = {
+            name = cfg.user;
+            group = cfg.group;
+            home = lib.mkForce "/var/lib/${cfg.user}";
+            createHome = true;
+            isSystemUser = true;
+            extraGroups = [ "keys" ];
+          };
+        };
+        users.groups = {
+          ${cfg.group} = {
+            name = cfg.group;
+          };
+        };
+      }
+      {
+        environment.systemPackages = lib.optionals (enabledInstances != {}) [ pkgs.restic ];
+
+        services.restic.backups =
+          let
+            mkRepositorySettings = name: instance: repository: {
+              "${name}_${repoSlugName repository.path}" = {
+                inherit (cfg) user;
+                repository = repository.path;
+
+                paths = instance.sourceDirectories;
+
+                passwordFile = toString instance.passphraseFile;
+
+                initialize = true;
+
+                inherit (repository) timerConfig;
+
+                pruneOpts = lib.mapAttrsToList (name: value:
+                  "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
+                ) instance.retention;
+
+                backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
+
+                backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
+              } // lib.attrsets.optionalAttrs (repository.extraSecrets != {}) {
+                environmentFile = shblib.replaceSecrets {
+                  userConfig = repository.extraSecrets;
+                  resultPath = "/var/lib/backup/${name}";
+                  generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} v);
+                };
+              } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
+                exclude = instance.excludePatterns;
+              };
+            };
+
+            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
+          in
+            lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances));
+
+        systemd.services =
+          let
+            mkRepositorySettings = name: instance: repository: {
+              "restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = {
+                Nice = cfg.performance.niceness;
+                IOSchedulingClass = cfg.performance.ioSchedulingClass;
+                IOSchedulingPriority = cfg.performance.ioPriority;
+              };
+            };
+            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
+          in
+            lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances));
+      }
+      {
+        environment.systemPackages = let
+          mkResticBinary = name: instance: repository: pkgs.writeShellScriptBin "restic-${name}_${repoSlugName repository.path}" ''
+            export RESTIC_PASSWORD_FILE=${instance.passphraseFile}
+            export RESTIC_REPOSITORY=${repository.path}
+            ${pkgs.restic}/bin/restic $@
+          '';
+          mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.repositories;
+        in
+          lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances);
+      }
+    ]);
+}

modules/blocks/restic/docs/default.md

@@ -1,33 +1,32 @@
-# Backup Block {#blocks-backup}
+# Restic Block {#blocks-restic}
 
-Defined in [`/modules/blocks/backup.nix`](@REPO@/modules/blocks/backup.nix).
+Defined in [`/modules/blocks/restic.nix`](@REPO@/modules/blocks/restic.nix).
 
-This block sets up backup jobs for Self Host Blocks.
+This block sets up a backup job using [Restic][restic].
 
-## Features {#blocks-backup-features}
+[restic]: https://restic.net/
-Two implementations for this block are provided:
-- [Restic](https://restic.net/)
-- [Borgmatic](https://torsion.org/borgmatic/)
 
-No integration tests are provided yet.
+## Contract {#blocks-restic-features}
+
+This block implements the [backup](contracts-backup.html) contract.
+
+Integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
 
 ## Usage {#blocks-backup-usage}
 
 ### One folder backed up to mounted hard drives {#blocks-backup-usage-one}
 
-The following snippet shows how to configure backup of 1 folder using the Restic implementation to 1
+The following snippet shows how to configure
-repository.
+the backup of 1 folder to 1 repository.
 
 Assumptions:
 - 1 hard drive pool is used for backup and is mounted on `/srv/pool1`.
 
 ```nix
-shb.backup.instances.myfolder = {
+shb.restic.instances.myfolder = {
   enable = true;
 
-  backend = "restic";
+  passphraseFile = "<path/to/passphrase>";
-
-  keySopsFile = ./secrets.yaml;
 
   repositories = [{
     path = "/srv/pool1/backups/myfolder";
@@ -56,36 +55,14 @@ shb.backup.instances.myfolder = {
 };
 ```
 
-The referenced Sops file must follow this structure:
+To be secure, the `passphraseFile` must contain a secret that is deployed out of band, otherwise it will be world-readable in the nix store.
+To achieve that, I recommend [sops](usage.html#usage-secrets) although other methods work great too.
 
-```yaml
+### One folder backed up to S3 {#blocks-restic-usage-remote}
-restic:
-    passphrases:
-        myfolder: <secret>
-```
-
-To generate a secret, use: `nix run nixpkgs#openssl -- rand -hex 64`.
-
-With the borgmatic implementation, the structure should be:
-
-```yaml
-borgmatic:
-    keys:
-        myfolder: |
-            BORG_KEY <key>
-    passphrases:
-        myfolder: <secret>
-```
-
-You can have both borgmatic and restic implementations working at the same time.
-
-### One folder backed up to S3 {#blocks-backup-usage-remote}
-
-> This is only supported by the Restic implementation. 
 
 Here we will only highlight the differences with the previous configuration.
 
-This assumes you have access to such a remote S3 store, for example by using Backblaze.
+This assumes you have access to such a remote S3 store, for example by using [Backblaze](https://www.backblaze.com/).
 
 ```diff
   shb.backup.instances.myfolder = {
@@ -97,31 +74,19 @@ This assumes you have access to such a remote S3 store, for example by using Bac
         OnCalendar = "00:00:00";
         RandomizedDelaySec = "3h";
       };
+
++     extraSecrets = {
++       AWS_ACCESS_KEY_ID="<path/to/access_key_id>";
++       AWS_SECRET_ACCESS_KEY="<path/to/secret_access_key>";
++     };
     }];
-
-
-+   environmentFile = true; # Needed for s3
   }
 ```
 
-The Sops file has a new required field:
+### Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
 
-```yaml
+The following snippet shows how to configure backup of any number of folders to 3 repositories,
-
+each happening at different times to avoid I/O contention.
-  restic:
-      passphrases:
-          myfolder: <secret>
-+     environmentfiles:
-+         myfolder: |-
-+             AWS_ACCESS_KEY_ID=<aws_key_id>
-+             AWS_SECRET_ACCESS_KEY=<aws_secret_key>
-```
-
-### Multiple folder to multiple destinations {#blocks-backup-usage-multiple}
-
-The following snippet shows how to configure backup of any number of folders using the Restic
-implementation to 3 repositories, each happening at different times to avoid contending for I/O
-time.
 
 We will also make sure to be able to re-use as much as the configuration as possible.
 
@@ -129,7 +94,7 @@ A few assumptions:
 - 2 hard drive pools used for backup are mounted respectively on `/srv/pool1` and `/srv/pool2`.
 - You have a backblaze account.
 
-First, let's define a variable to hold all our repositories you want to back up to:
+First, let's define a variable to hold all the repositories we want to back up to:
 
 ```nix
 repos = [
@@ -209,19 +174,38 @@ below) is the former splits the backups into sub-folders on the repositories.
 shb.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"];
 ```
 
-## Demo {#blocks-backup-demo}
+## Demo {#blocks-restic-demo}
 
 [WIP]
 
-## Monitoring {#blocks-backup-monitoring}
+## Monitoring {#blocks-restic-monitoring}
 
 [WIP]
 
-## Maintenance {#blocks-backup-maintenance}
+## Maintenance {#blocks-restic-maintenance}
 
-[WIP]
+One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
 
-## Options Reference {#blocks-backup-options}
+In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:
+
+```bash
+restic-myfolder1_srv_pool1_backups
+restic-myfolder1_srv_pool2_backups
+restic-myfolder1_s3_s3.us-west-000.backblazeb2.com_backups
+restic-myfolder2_srv_pool1_backups
+restic-myfolder2_srv_pool2_backups
+restic-myfolder2_s3_s3.us-west-000.backblazeb2.com_backups
+```
+
+Discovering those is easy thanks to tab-completion.
+
+One can then restore a backup with:
+
+```bash
+restic-myfolder1_srv_pool1_backups restore latest -t /
+```
+
+## Options Reference {#blocks-restic-options}
 
 ```{=include=} options
 id-prefix: blocks-backup-options-

modules/contracts/backup.nix

@@ -0,0 +1,61 @@
+{ lib, ... }:
+lib.types.submodule {
+  freeformType = lib.types.anything;
+
+  options = {
+    user = lib.mkOption {
+      description = "Unix user doing the backups.";
+      type = lib.types.str;
+      default = "backup";
+    };
+
+    group = lib.mkOption {
+      description = "Unix group doing the backups.";
+      type = lib.types.str;
+      default = "backup";
+    };
+
+    sourceDirectories = lib.mkOption {
+      description = "Directories to backup.";
+      type = lib.types.nonEmptyListOf lib.types.str;
+    };
+
+    excludePatterns = lib.mkOption {
+      description = "Patterns to exclude.";
+      type = lib.types.listOf lib.types.str;
+      default = [];
+    };
+
+    retention = lib.mkOption {
+      description = "Backup files retention.";
+      type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]);
+      default = {
+        keep_within = "1d";
+        keep_hourly = 24;
+        keep_daily = 7;
+        keep_weekly = 4;
+        keep_monthly = 6;
+      };
+    };
+
+    hooks = lib.mkOption {
+      description = "Hooks to run around the backup.";
+      default = {};
+      type = lib.types.submodule {
+        options = {
+          before_backup = lib.mkOption {
+            description = "Hooks to run before backup";
+            type = lib.types.listOf lib.types.str;
+            default = [];
+          };
+
+          after_backup = lib.mkOption {
+            description = "Hooks to run after backup";
+            type = lib.types.listOf lib.types.str;
+            default = [];
+          };
+        };
+      };
+    };
+  };
+}

modules/contracts/backup/docs/default.md

@@ -0,0 +1,56 @@
+# Backup Contract {#backup-contract}
+
+This NixOS contract represents a backup job
+that will backup one or more files or directories
+at a regular schedule.
+
+## Contract Reference {#backup-contract-options}
+
+These are all the options that are expected to exist for this contract to be respected.
+
+```{=include=} options
+id-prefix: contracts-backup-options-
+list-id: selfhostblocks-options
+source: @OPTIONS_JSON@
+```
+
+## Usage {#backup-contract-usage}
+
+A service that can be backed up will provide a `backup` option, like for the [Vaultwarden service][vaultwarden-service-backup].
+What this option defines is an implementation detail of that service
+but it will at least define what directories to backup
+and possibly hooks to run before or after the backup job runs.
+
+[vaultwarden-service-backup]: services-vaultwarden.html#services-vaultwarden-options-shb.vaultwarden.backup
+
+```nix
+shb.<service>.backup
+```
+
+Let's assume a module implementing this contract is available under the `shb.<backup_impl>` variable.
+Then, to actually backup the service, one would write:
+
+```nix
+shb.<backup_impl>.instances."<service>" = shb.<service>.backup // {
+  enable = true;
+
+  // Options specific to backup_impl
+};
+```
+
+Then, for extra caution, a second backup could be made using another module `shb.<backup_impl_2>`:
+
+```nix
+shb.<backup_impl_2>.instances."<service>" = shb.<service>.backup // {
+  enable = true;
+
+  // Options specific to backup_impl_2
+};
+```
+
+## Provided Implementations {#backup-contract-impl}
+
+One implementation is provided out of the box:
+- [Restic block](blocks-restic.html).
+
+A second one based on `borgbackup` is in progress.

modules/contracts/backup/dummyModule.nix

@@ -0,0 +1,10 @@
+{ pkgs, lib, ... }:
+let
+  contracts = pkgs.callPackage ../. {};
+in
+{
+  options.shb.contracts.backup = lib.mkOption {
+    description = "Contract for backups.";
+    type = contracts.backup;
+  };
+}

modules/contracts/default.nix

@@ -1,5 +1,6 @@
 { lib }:
 {
+  backup = import ./backup.nix { inherit lib; };
   mount = import ./mount.nix { inherit lib; };
   ssl = import ./ssl.nix { inherit lib; };
 }

modules/services/vaultwarden.nix

@@ -114,10 +114,26 @@ in
       default = { path = dataFolder; };
     };
 
-    backupConfig = lib.mkOption {
+    backup = lib.mkOption {
-      type = lib.types.nullOr lib.types.anything;
+      type = contracts.backup;
-      description = "Backup configuration of Vaultwarden.";
+      description = ''
-      default = null;
+        Backup configuration. This is an output option.
+
+        Use it to initialize a block implementing the "backup" contract.
+        For example, with the restic block:
+
+        ```
+        shb.restic.instances."vaultwarden" = {
+          poolName = "root";
+        } // config.shb.vaultwarden.backup;
+        ```
+      '';
+      readOnly = true;
+      default = {
+        sourceDirectories = [
+          dataFolder
+        ];
+      };
     };
 
     debug = lib.mkOption {
@@ -217,14 +233,6 @@ in
       members = [ "backup" ];
     };
 
-    shb.backup.instances.vaultwarden = lib.mkIf (cfg.backupConfig != null) (
-      cfg.backupConfig //
-      {
-        sourceDirectories = [
-          config.services.vaultwarden.config.DATA_FOLDER
-        ];
-      });
-
     # TODO: make this work.
     # It does not work because it leads to infinite recursion.
     # ${cfg.mount}.path = dataFolder;

modules/services/vaultwarden/docs/default.md

@@ -91,3 +91,25 @@ Integration with the ZFS block allows to automatically create the relevant datas
 shb.zfs.datasets."vaultwarden" = config.shb.vaultwarden.mount;
 shb.zfs.datasets."postgresql".path = "/var/lib/postgresql";
 ```
+
+## Maintenance {#services-vaultwarden-maintenance}
+
+No command-line tool is provided to administer Vaultwarden.
+
+Instead, the admin section can be found at the `/admin` endpoint.
+
+## Debug {#services-backup-debug}
+
+In case of an issue, check the logs of the `vaultwarden.service` systemd service.
+
+Enable verbose logging by setting the `shb.vaultwarden.debug` boolean to `true`.
+
+Access the database with `sudo -u vaultwarden psql`.
+
+## Options Reference {#services-vaultwarden-options}
+
+```{=include=} options
+id-prefix: services-vaultwarden-options-
+list-id: selfhostblocks-vaultwarden-options
+source: @OPTIONS_JSON@
+```

test/blocks/restic.nix

@@ -0,0 +1,156 @@
+{ pkgs, lib, ... }:
+let
+  pkgs' = pkgs;
+
+  testLib = pkgs.callPackage ../common.nix {};
+
+  base = testLib.base [
+    ../../modules/blocks/restic.nix
+  ];
+in
+{
+  backupAndRestore = pkgs.testers.runNixOSTest {
+    name = "restic_backupAndRestore";
+
+    nodes.machine = {
+      imports = ( testLib.baseImports pkgs' ) ++ [
+        ../../modules/blocks/restic.nix
+      ];
+
+      shb.restic = {
+        user = "root";
+        group = "root";
+      };
+      shb.restic.instances."testinstance" = {
+        enable = true;
+
+        passphraseFile = pkgs.writeText "passphrase" "PassPhrase";
+
+        sourceDirectories = [
+          "/opt/files/A"
+          "/opt/files/B"
+        ];
+
+        repositories = [
+          {
+            path = "/opt/repos/A";
+            timerConfig = {
+              OnCalendar = "00:00:00";
+              RandomizedDelaySec = "5h";
+            };
+          }
+          {
+            path = "/opt/repos/B";
+            timerConfig = {
+              OnCalendar = "00:00:00";
+              RandomizedDelaySec = "5h";
+            };
+          }
+        ];
+      };
+    };
+
+    extraPythonPackages = p: [ p.dictdiffer ];
+    skipTypeCheck = true;
+
+    testScript = { nodes, ... }: let
+      instanceCfg = nodes.machine.shb.restic.instances."testinstance";
+    in ''
+    from dictdiffer import diff
+
+    def list_files(dir):
+        files_and_content = {}
+
+        files = machine.succeed(f"""
+        find {dir} -type f
+        """).split("\n")[:-1]
+
+        for f in files:
+            content = machine.succeed(f"""
+            cat {f}
+            """).strip()
+            files_and_content[f] = content
+
+        return files_and_content
+
+    def assert_files(dir, files):
+        result = list(diff(list_files(dir), files))
+        if len(result) > 0:
+            raise Exception("Unexpected files:", result)
+
+    with subtest("Create initial content"):
+        machine.succeed("""
+        mkdir -p /opt/files/A
+        mkdir -p /opt/files/B
+        mkdir -p /opt/repos/A
+        mkdir -p /opt/repos/B
+
+        echo repoA_fileA_1 > /opt/files/A/fileA
+        echo repoA_fileB_1 > /opt/files/A/fileB
+        echo repoB_fileA_1 > /opt/files/B/fileA
+        echo repoB_fileB_1 > /opt/files/B/fileB
+
+        # chown :backup -R /opt/files
+        """)
+
+        assert_files("/opt/files", {
+            '/opt/files/B/fileA': 'repoB_fileA_1',
+            '/opt/files/B/fileB': 'repoB_fileB_1',
+            '/opt/files/A/fileA': 'repoA_fileA_1',
+            '/opt/files/A/fileB': 'repoA_fileB_1',
+        })
+
+    with subtest("First backup in repo A"):
+        machine.succeed("systemctl start restic-backups-testinstance_opt_repos_A")
+
+    with subtest("New content"):
+        machine.succeed("""
+        echo repoA_fileA_2 > /opt/files/A/fileA
+        echo repoA_fileB_2 > /opt/files/A/fileB
+        echo repoB_fileA_2 > /opt/files/B/fileA
+        echo repoB_fileB_2 > /opt/files/B/fileB
+        """)
+
+        assert_files("/opt/files", {
+            '/opt/files/B/fileA': 'repoB_fileA_2',
+            '/opt/files/B/fileB': 'repoB_fileB_2',
+            '/opt/files/A/fileA': 'repoA_fileA_2',
+            '/opt/files/A/fileB': 'repoA_fileB_2',
+        })
+
+    with subtest("Second backup in repo B"):
+        machine.succeed("systemctl start restic-backups-testinstance_opt_repos_B")
+
+    with subtest("Delete content"):
+        machine.succeed("""
+        rm -r /opt/files/A /opt/files/B
+        """)
+
+        assert_files("/opt/files", {})
+
+    with subtest("Restore initial content from repo A"):
+        machine.succeed("""
+        restic-testinstance_opt_repos_A restore latest -t /
+        """)
+
+        assert_files("/opt/files", {
+            '/opt/files/B/fileA': 'repoB_fileA_1',
+            '/opt/files/B/fileB': 'repoB_fileB_1',
+            '/opt/files/A/fileA': 'repoA_fileA_1',
+            '/opt/files/A/fileB': 'repoA_fileB_1',
+        })
+
+    with subtest("Restore initial content from repo B"):
+        machine.succeed("""
+        restic-testinstance_opt_repos_B restore latest -t /
+        """)
+
+        assert_files("/opt/files", {
+            '/opt/files/B/fileA': 'repoB_fileA_2',
+            '/opt/files/B/fileB': 'repoB_fileB_2',
+            '/opt/files/A/fileA': 'repoA_fileA_2',
+            '/opt/files/A/fileB': 'repoA_fileB_2',
+        })
+    '';
+  };
+}

test/common.nix

@@ -1,6 +1,12 @@
 {
   lib,
 }:
+let
+  baseImports = pkgs: [
+    (pkgs.path + "/nixos/modules/profiles/headless.nix")
+    (pkgs.path + "/nixos/modules/profiles/qemu-guest.nix")
+  ];
+in
 {
   accessScript = {
     subdomain
@@ -92,21 +98,24 @@
         ${indent 4 script}
       '');
 
+  inherit baseImports;
+
   base = pkgs: additionalModules: {
-    imports = [
+    imports =
-      (pkgs.path + "/nixos/modules/profiles/headless.nix")
+      ( baseImports pkgs )
-      (pkgs.path + "/nixos/modules/profiles/qemu-guest.nix")
+      ++ [
-      # TODO: replace this option by the backup contract
+        # TODO: replace this option by the backup contract
-      {
+        {
-        options = {
+          options = {
-          shb.backup = lib.mkOption { type = lib.types.anything; };
+            shb.backup = lib.mkOption { type = lib.types.anything; };
-        };
+          };
-      }
+        }
-      # TODO: replace postgresql.nix and authelia.nix by the sso contract
+        # TODO: replace postgresql.nix and authelia.nix by the sso contract
-      ../modules/blocks/postgresql.nix
+        ../modules/blocks/postgresql.nix
-      ../modules/blocks/authelia.nix
+        ../modules/blocks/authelia.nix
-      ../modules/blocks/nginx.nix
+        ../modules/blocks/nginx.nix
-    ] ++ additionalModules;
+      ]
+      ++ additionalModules;
 
     # Nginx port.
     networking.firewall.allowedTCPPorts = [ 80 443 ];