diff --git a/docs/blocks.md b/docs/blocks.md
index 3849fd2..9eff596 100644
--- a/docs/blocks.md
+++ b/docs/blocks.md
@@ -34,8 +34,8 @@ Not all blocks are yet documented. You can find all available blocks [in the rep
modules/blocks/ssl/docs/default.md
```
-```{=include=} chapters html:into-file=//blocks-backup.html
-modules/blocks/backup/docs/default.md
+```{=include=} chapters html:into-file=//blocks-restic.html
+modules/blocks/restic/docs/default.md
```
```{=include=} chapters html:into-file=//blocks-monitoring.html
diff --git a/docs/contracts.md b/docs/contracts.md
index 8af45dc..d241b90 100644
--- a/docs/contracts.md
+++ b/docs/contracts.md
@@ -19,12 +19,19 @@ as possible, reducing the quite thick layer that it is now.
Provided contracts are:
-- [SSL generator contract](contracts-ssl.html) to generate SSL certificates. Two implementations are provided: self-signed and Let's Encrypt.
+- [SSL generator contract](contracts-ssl.html) to generate SSL certificates.
+ Two implementations are provided: self-signed and Let's Encrypt.
+- [Backup contract](contracts-backup.html) to backup directories.
+ This contract allows to backup multiple times the same directories for extra protection.
```{=include=} chapters html:into-file=//contracts-ssl.html
modules/contracts/ssl/docs/default.md
```
+```{=include=} chapters html:into-file=//contracts-backup.html
+modules/contracts/backup/docs/default.md
+```
+
## Why do we need this new concept? {#contracts-why}
Currently in nixpkgs, every module needing access to a shared resource must implement the logic
diff --git a/docs/default.nix b/docs/default.nix
index e02ffe8..bfc6989 100644
--- a/docs/default.nix
+++ b/docs/default.nix
@@ -67,7 +67,9 @@ let
};
optionsDocs = buildOptionsDocs {
- modules = allModules ++ [ scrubbedModule ];
+ modules = allModules ++ [
+ scrubbedModule
+ ];
variablelistId = "selfhostblocks-options";
includeModuleSystemOptions = false;
};
@@ -134,10 +136,10 @@ in stdenv.mkDerivation {
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs ../modules/blocks/ssl.nix}/share/doc/nixos/options.json
- substituteInPlace ./modules/blocks/backup/docs/default.md \
+ substituteInPlace ./modules/blocks/restic/docs/default.md \
--replace \
'@OPTIONS_JSON@' \
- ${individualModuleOptionsDocs ../modules/blocks/backup.nix}/share/doc/nixos/options.json
+ ${individualModuleOptionsDocs ../modules/blocks/restic.nix}/share/doc/nixos/options.json
substituteInPlace ./modules/services/nextcloud-server/docs/default.md \
--replace \
@@ -149,6 +151,11 @@ in stdenv.mkDerivation {
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs ../modules/services/vaultwarden.nix}/share/doc/nixos/options.json
+ substituteInPlace ./modules/contracts/backup/docs/default.md \
+ --replace \
+ '@OPTIONS_JSON@' \
+ ${individualModuleOptionsDocs ../modules/contracts/backup/dummyModule.nix}/share/doc/nixos/options.json
+
substituteInPlace ./modules/contracts/ssl/docs/default.md \
--replace \
'@OPTIONS_JSON@' \
diff --git a/flake.lock b/flake.lock
index 08c9431..5431f5e 100644
--- a/flake.lock
+++ b/flake.lock
@@ -49,38 +49,6 @@
"type": "github"
}
},
- "nixpkgs-stable": {
- "locked": {
- "lastModified": 1716655032,
- "narHash": "sha256-kQ25DAiCGigsNR/Quxm3v+JGXAEXZ8I7RAF4U94bGzE=",
- "owner": "NixOS",
- "repo": "nixpkgs",
- "rev": "59a450646ec8ee0397f5fa54a08573e8240eb91f",
- "type": "github"
- },
- "original": {
- "owner": "NixOS",
- "ref": "release-23.11",
- "repo": "nixpkgs",
- "type": "github"
- }
- },
- "nixpkgs_2": {
- "locked": {
- "lastModified": 1716651315,
- "narHash": "sha256-iMgzIeedMqf30TXZ439zW3Yvng1Xm9QTGO+ZwG1IWSw=",
- "owner": "NixOS",
- "repo": "nixpkgs",
- "rev": "c5187508b11177ef4278edf19616f44f21cc8c69",
- "type": "github"
- },
- "original": {
- "owner": "NixOS",
- "ref": "nixpkgs-unstable",
- "repo": "nixpkgs",
- "type": "github"
- }
- },
"nmdsrc": {
"flake": false,
"locked": {
@@ -102,27 +70,7 @@
"flake-utils": "flake-utils",
"nix-flake-tests": "nix-flake-tests",
"nixpkgs": "nixpkgs",
- "nmdsrc": "nmdsrc",
- "sops-nix": "sops-nix"
- }
- },
- "sops-nix": {
- "inputs": {
- "nixpkgs": "nixpkgs_2",
- "nixpkgs-stable": "nixpkgs-stable"
- },
- "locked": {
- "lastModified": 1716692524,
- "narHash": "sha256-sALodaA7Zkp/JD6ehgwc0UCBrSBfB4cX66uFGTsqeFU=",
- "owner": "Mic92",
- "repo": "sops-nix",
- "rev": "962797a8d7f15ed7033031731d0bb77244839960",
- "type": "github"
- },
- "original": {
- "owner": "Mic92",
- "repo": "sops-nix",
- "type": "github"
+ "nmdsrc": "nmdsrc"
}
},
"systems": {
diff --git a/flake.nix b/flake.nix
index cb21bfb..fd69f8a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -36,12 +36,12 @@
allModules = [
modules/blocks/authelia.nix
- modules/blocks/backup.nix
modules/blocks/davfs.nix
modules/blocks/ldap.nix
modules/blocks/monitoring.nix
modules/blocks/nginx.nix
modules/blocks/postgresql.nix
+ modules/blocks/restic.nix
modules/blocks/ssl.nix
modules/blocks/tinyproxy.nix
modules/blocks/vpn.nix
@@ -60,6 +60,7 @@
# Only used for documentation.
contractDummyModules = [
+ modules/contracts/backup/dummyModule.nix
modules/contracts/ssl/dummyModule.nix
];
in
@@ -133,6 +134,7 @@
// (vm_test "ldap" ./test/blocks/ldap.nix)
// (vm_test "lib" ./test/blocks/lib.nix)
// (vm_test "postgresql" ./test/blocks/postgresql.nix)
+ // (vm_test "restic" ./test/blocks/restic.nix)
// (vm_test "ssl" ./test/blocks/ssl.nix)
);
}
diff --git a/modules/blocks/backup.nix b/modules/blocks/borgbackup.nix
similarity index 69%
rename from modules/blocks/backup.nix
rename to modules/blocks/borgbackup.nix
index 3e57d58..ebf0a04 100644
--- a/modules/blocks/backup.nix
+++ b/modules/blocks/borgbackup.nix
@@ -1,30 +1,32 @@
{ config, pkgs, lib, utils, ... }:
let
- cfg = config.shb.backup;
+ cfg = config.shb.borgbackup;
instanceOptions = {
- enable = lib.mkEnableOption "shb backup instance";
-
- backend = lib.mkOption {
- description = "What program to use to make the backups.";
- type = lib.types.enum [ "borgmatic" "restic" ];
- example = "borgmatic";
- };
+ enable = lib.mkEnableOption "shb borgbackup";
keySopsFile = lib.mkOption {
- description = "Sops file that holds this instance's Borgmatic repository key and passphrase.";
+ description = "Sops file that holds this instance's repository key and passphrase.";
type = lib.types.path;
example = "secrets/backup.yaml";
};
+ encryptionKeyFile = lib.mkOption {
+ description = "Encryption key for the backup.";
+ type = lib.types.path;
+ };
+
+ encryption_passcommand = "cat /run/secrets/borgmatic/passphrases/${if isNull instance.secretName then name else instance.secretName}";
+ borg_keys_directory = "/run/secrets/borgmatic/keys";
+
sourceDirectories = lib.mkOption {
- description = "Borgmatic source directories.";
+ description = "Source directories.";
type = lib.types.nonEmptyListOf lib.types.str;
};
excludePatterns = lib.mkOption {
- description = "Borgmatic exclude patterns.";
+ description = "Exclude patterns.";
type = lib.types.listOf lib.types.str;
default = [];
};
@@ -74,7 +76,7 @@ let
};
consistency = lib.mkOption {
- description = "Consistency frequency options. Only applicable for borgmatic";
+ description = "Consistency frequency options.";
type = lib.types.attrsOf lib.types.nonEmptyStr;
default = {};
example = {
@@ -84,7 +86,7 @@ let
};
hooks = lib.mkOption {
- description = "Borgmatic hooks.";
+ description = "Hooks to run before or after the backup.";
default = {};
type = lib.types.submodule {
options = {
@@ -115,14 +117,7 @@ let
in
{
- options.shb.backup = {
- onlyOnAC = lib.mkOption {
- description = "Run backups only if AC power is plugged in.";
- default = true;
- example = false;
- type = lib.types.bool;
- };
-
+ options.shb.borgbackup = {
user = lib.mkOption {
description = "Unix user doing the backups.";
type = lib.types.str;
@@ -163,12 +158,12 @@ in
};
ioSchedulingClass = lib.mkOption {
type = lib.types.enum [ "idle" "best-effort" "realtime" ];
- description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
+ description = "ionice scheduling class, defaults to best-effort IO.";
default = "best-effort";
};
ioPriority = lib.mkOption {
type = lib.types.nullOr (lib.types.ints.between 0 7);
- description = "ionice priority, defaults to 7 for lowest priority IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
+ description = "ionice priority, defaults to 7 for lowest priority IO.";
default = 7;
};
};
@@ -179,8 +174,6 @@ in
config = lib.mkIf (cfg.instances != {}) (
let
enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
- borgmaticInstances = lib.attrsets.filterAttrs (k: i: i.backend == "borgmatic") enabledInstances;
- resticInstances = lib.attrsets.filterAttrs (k: i: i.backend == "restic") enabledInstances;
in lib.mkMerge [
# Secrets configuration
{
@@ -234,13 +227,13 @@ in
}
# Borgmatic configuration
{
- systemd.timers.borgmatic = lib.mkIf (borgmaticInstances != {}) {
+ systemd.timers.borgmatic = lib.mkIf (enabledInstances != {}) {
timerConfig = {
OnCalendar = "hourly";
};
};
- systemd.services.borgmatic = lib.mkIf (borgmaticInstances != {}) {
+ systemd.services.borgmatic = lib.mkIf (enabledInstances != {}) {
serviceConfig = {
User = cfg.user;
Group = cfg.group;
@@ -252,10 +245,10 @@ in
};
};
- systemd.packages = lib.mkIf (borgmaticInstances != {}) [ pkgs.borgmatic ];
+ systemd.packages = lib.mkIf (enabledInstances != {}) [ pkgs.borgmatic ];
environment.systemPackages = (
lib.optionals cfg.borgServer [ pkgs.borgbackup ]
- ++ lib.optionals (borgmaticInstances != {}) [ pkgs.borgbackup pkgs.borgmatic ]
+ ++ lib.optionals (enabledInstances != {}) [ pkgs.borgbackup pkgs.borgmatic ]
);
environment.etc =
@@ -272,7 +265,7 @@ in
});
storage = {
- encryption_passcommand = "cat /run/secrets/borgmatic/passphrases/${if isNull instance.secretName then name else instance.secretName}";
+ encryption_passcommand = "cat ${instance.encryptionKeyFile}";
borg_keys_directory = "/run/secrets/borgmatic/keys";
};
@@ -296,57 +289,7 @@ in
};
};
in
- lib.mkMerge (lib.attrsets.mapAttrsToList mkSettings borgmaticInstances);
- }
- # Restic configuration
- {
- environment.systemPackages = lib.optionals (resticInstances != {}) [ pkgs.restic ];
-
- services.restic.backups =
- let
- mkRepositorySettings = name: instance: repository: {
- "${name}_${repoSlugName repository.path}" = {
- inherit (cfg) user;
- repository = repository.path;
-
- paths = instance.sourceDirectories;
-
- passwordFile = "/run/secrets/${instance.backend}/passphrases/${name}";
-
- initialize = true;
-
- inherit (repository) timerConfig;
-
- pruneOpts = lib.mapAttrsToList (name: value:
- "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
- ) instance.retention;
-
- backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
-
- backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
- } // lib.attrsets.optionalAttrs (instance.environmentFile) {
- environmentFile = "/run/secrets/${instance.backend}/environmentfiles/${name}";
- } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
- exclude = instance.excludePatterns;
- };
- };
-
- mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
- in
- lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings resticInstances));
-
- systemd.services =
- let
- mkRepositorySettings = name: instance: repository: {
- "restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = {
- Nice = cfg.performance.niceness;
- IOSchedulingClass = cfg.performance.ioSchedulingClass;
- IOSchedulingPriority = cfg.performance.ioPriority;
- };
- };
- mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
- in
- lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings resticInstances));
+ lib.mkMerge (lib.attrsets.mapAttrsToList mkSettings enabledInstances);
}
]);
}
diff --git a/modules/blocks/restic.nix b/modules/blocks/restic.nix
new file mode 100644
index 0000000..24014b3
--- /dev/null
+++ b/modules/blocks/restic.nix
@@ -0,0 +1,238 @@
+{ config, pkgs, lib, utils, ... }:
+
+let
+ cfg = config.shb.restic;
+
+ shblib = pkgs.callPackage ../../lib {};
+
+ instanceOptions = {
+ enable = lib.mkEnableOption "shb restic";
+
+ passphraseFile = lib.mkOption {
+ description = "Encryption key for the backups.";
+ type = lib.types.path;
+ };
+
+ sourceDirectories = lib.mkOption {
+ description = "Source directories.";
+ type = lib.types.nonEmptyListOf lib.types.str;
+ };
+
+ excludePatterns = lib.mkOption {
+ description = "Exclude patterns.";
+ type = lib.types.listOf lib.types.str;
+ default = [];
+ };
+
+ repositories = lib.mkOption {
+ description = "Repositories to back this instance to.";
+ type = lib.types.nonEmptyListOf (lib.types.submodule {
+ options = {
+ path = lib.mkOption {
+ type = lib.types.str;
+ description = "Repository location";
+ };
+
+ extraSecrets = lib.mkOption {
+ type = lib.types.attrsOf shblib.secretFileType;
+ default = {};
+ description = ''
+ Extra secrets needed to access the repository where the backups will be stored.
+
+ See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example.
+ '';
+ example = lib.literalExpression ''
+ {
+ AWS_ACCESS_KEY_ID = <path/to/secret>;
+ AWS_SECRET_ACCESS_KEY = <path/to/secret>;
+ }
+ '';
+ };
+
+ timerConfig = lib.mkOption {
+ type = lib.types.attrsOf utils.systemdUtils.unitOptions.unitOption;
+ default = {
+ OnCalendar = "daily";
+ Persistent = true;
+ };
+ description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
+ example = {
+ OnCalendar = "00:05";
+ RandomizedDelaySec = "5h";
+ Persistent = true;
+ };
+ };
+ };
+ });
+ };
+
+ retention = lib.mkOption {
+ description = "For how long to keep backup files.";
+ type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]);
+ default = {
+ keep_within = "1d";
+ keep_hourly = 24;
+ keep_daily = 7;
+ keep_weekly = 4;
+ keep_monthly = 6;
+ };
+ };
+
+ hooks = lib.mkOption {
+ description = "Hooks to run before or after the backup.";
+ default = {};
+ type = lib.types.submodule {
+ options = {
+ before_backup = lib.mkOption {
+ description = "Hooks to run before backup";
+ type = lib.types.listOf lib.types.str;
+ default = [];
+ };
+
+ after_backup = lib.mkOption {
+ description = "Hooks to run after backup";
+ type = lib.types.listOf lib.types.str;
+ default = [];
+ };
+ };
+ };
+ };
+ };
+
+ repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (lib.strings.removePrefix "/" name);
+in
+{
+ options.shb.restic = {
+ user = lib.mkOption {
+ description = "Unix user doing the backups.";
+ type = lib.types.str;
+ default = "backup";
+ };
+
+ group = lib.mkOption {
+ description = "Unix group doing the backups.";
+ type = lib.types.str;
+ default = "backup";
+ };
+
+ instances = lib.mkOption {
+ description = "Each instance is a backup setting";
+ default = {};
+ type = lib.types.attrsOf (lib.types.submodule {
+ options = instanceOptions;
+ });
+ };
+
+ # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23
+ performance = lib.mkOption {
+ description = "Reduce performance impact of backup jobs.";
+ default = {};
+ type = lib.types.submodule {
+ options = {
+ niceness = lib.mkOption {
+ type = lib.types.ints.between (-20) 19;
+ description = "nice priority adjustment, defaults to 15 for ~20% CPU time of normal-priority process";
+ default = 15;
+ };
+ ioSchedulingClass = lib.mkOption {
+ type = lib.types.enum [ "idle" "best-effort" "realtime" ];
+ description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
+ default = "best-effort";
+ };
+ ioPriority = lib.mkOption {
+ type = lib.types.nullOr (lib.types.ints.between 0 7);
+ description = "ionice priority, defaults to 7 for lowest priority IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
+ default = 7;
+ };
+ };
+ };
+ };
+ };
+
+ config = lib.mkIf (cfg.instances != {}) (
+ let
+ enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
+ in lib.mkMerge [
+ {
+ users.users = {
+ ${cfg.user} = {
+ name = cfg.user;
+ group = cfg.group;
+ home = lib.mkForce "/var/lib/${cfg.user}";
+ createHome = true;
+ isSystemUser = true;
+ extraGroups = [ "keys" ];
+ };
+ };
+ users.groups = {
+ ${cfg.group} = {
+ name = cfg.group;
+ };
+ };
+ }
+ {
+ environment.systemPackages = lib.optionals (enabledInstances != {}) [ pkgs.restic ];
+
+ services.restic.backups =
+ let
+ mkRepositorySettings = name: instance: repository: {
+ "${name}_${repoSlugName repository.path}" = {
+ inherit (cfg) user;
+ repository = repository.path;
+
+ paths = instance.sourceDirectories;
+
+ passwordFile = toString instance.passphraseFile;
+
+ initialize = true;
+
+ inherit (repository) timerConfig;
+
+ pruneOpts = lib.mapAttrsToList (name: value:
+ "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
+ ) instance.retention;
+
+ backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
+
+ backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
+ } // lib.attrsets.optionalAttrs (repository.extraSecrets != {}) {
+ environmentFile = shblib.replaceSecrets {
+ userConfig = repository.extraSecrets;
+ resultPath = "/var/lib/backup/${name}";
+ generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} v);
+ };
+ } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
+ exclude = instance.excludePatterns;
+ };
+ };
+
+ mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
+ in
+ lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances));
+
+ systemd.services =
+ let
+ mkRepositorySettings = name: instance: repository: {
+ "restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = {
+ Nice = cfg.performance.niceness;
+ IOSchedulingClass = cfg.performance.ioSchedulingClass;
+ IOSchedulingPriority = cfg.performance.ioPriority;
+ };
+ };
+ mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
+ in
+ lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances));
+ }
+ {
+ environment.systemPackages = let
+ mkResticBinary = name: instance: repository: pkgs.writeShellScriptBin "restic-${name}_${repoSlugName repository.path}" ''
+ export RESTIC_PASSWORD_FILE=${instance.passphraseFile}
+ export RESTIC_REPOSITORY=${repository.path}
+ ${pkgs.restic}/bin/restic $@
+ '';
+ mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.repositories;
+ in
+ lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances);
+ }
+ ]);
+}
diff --git a/modules/blocks/backup/docs/default.md b/modules/blocks/restic/docs/default.md
similarity index 60%
rename from modules/blocks/backup/docs/default.md
rename to modules/blocks/restic/docs/default.md
index c431b4f..7e09165 100644
--- a/modules/blocks/backup/docs/default.md
+++ b/modules/blocks/restic/docs/default.md
@@ -1,33 +1,32 @@
-# Backup Block {#blocks-backup}
+# Restic Block {#blocks-restic}
-Defined in [`/modules/blocks/backup.nix`](@REPO@/modules/blocks/backup.nix).
+Defined in [`/modules/blocks/restic.nix`](@REPO@/modules/blocks/restic.nix).
-This block sets up backup jobs for Self Host Blocks.
+This block sets up a backup job using [Restic][restic].
-## Features {#blocks-backup-features}
-Two implementations for this block are provided:
-- [Restic](https://restic.net/)
-- [Borgmatic](https://torsion.org/borgmatic/)
+[restic]: https://restic.net/
-No integration tests are provided yet.
+## Contract {#blocks-restic-features}
+
+This block implements the [backup](contracts-backup.html) contract.
+
+Integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
## Usage {#blocks-backup-usage}
### One folder backed up to mounted hard drives {#blocks-backup-usage-one}
-The following snippet shows how to configure backup of 1 folder using the Restic implementation to 1
-repository.
+The following snippet shows how to configure
+the backup of 1 folder to 1 repository.
Assumptions:
- 1 hard drive pool is used for backup and is mounted on `/srv/pool1`.
```nix
-shb.backup.instances.myfolder = {
+shb.restic.instances.myfolder = {
enable = true;
- backend = "restic";
-
- keySopsFile = ./secrets.yaml;
+ passphraseFile = "<path/to/passphrase>";
repositories = [{
path = "/srv/pool1/backups/myfolder";
@@ -56,36 +55,14 @@ shb.backup.instances.myfolder = {
};
```
-The referenced Sops file must follow this structure:
+To be secure, the `passphraseFile` must contain a secret that is deployed out of band, otherwise it will be world-readable in the nix store.
+To achieve that, I recommend [sops](usage.html#usage-secrets) although other methods work great too.
-```yaml
-restic:
- passphrases:
- myfolder: <secret>
-```
-
-To generate a secret, use: `nix run nixpkgs#openssl -- rand -hex 64`.
-
-With the borgmatic implementation, the structure should be:
-
-```yaml
-borgmatic:
- keys:
- myfolder: |
- BORG_KEY <key>
- passphrases:
- myfolder: <secret>
-```
-
-You can have both borgmatic and restic implementations working at the same time.
-
-### One folder backed up to S3 {#blocks-backup-usage-remote}
-
-> This is only supported by the Restic implementation.
+### One folder backed up to S3 {#blocks-restic-usage-remote}
Here we will only highlight the differences with the previous configuration.
-This assumes you have access to such a remote S3 store, for example by using Backblaze.
+This assumes you have access to such a remote S3 store, for example by using [Backblaze](https://www.backblaze.com/).
```diff
shb.backup.instances.myfolder = {
@@ -97,31 +74,19 @@ This assumes you have access to such a remote S3 store, for example by using Bac
OnCalendar = "00:00:00";
RandomizedDelaySec = "3h";
};
+
++ extraSecrets = {
++ AWS_ACCESS_KEY_ID="<path/to/access_key_id>";
++ AWS_SECRET_ACCESS_KEY="<path/to/secret_access_key>";
++ };
}];
-
-
-+ environmentFile = true; # Needed for s3
}
```
-The Sops file has a new required field:
+### Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
-```yaml
-
- restic:
- passphrases:
- myfolder: <secret>
-+ environmentfiles:
-+ myfolder: |-
-+ AWS_ACCESS_KEY_ID=<aws_key_id>
-+ AWS_SECRET_ACCESS_KEY=<aws_secret_key>
-```
-
-### Multiple folder to multiple destinations {#blocks-backup-usage-multiple}
-
-The following snippet shows how to configure backup of any number of folders using the Restic
-implementation to 3 repositories, each happening at different times to avoid contending for I/O
-time.
+The following snippet shows how to configure backup of any number of folders to 3 repositories,
+each happening at different times to avoid I/O contention.
We will also make sure to be able to re-use as much as the configuration as possible.
@@ -129,7 +94,7 @@ A few assumptions:
- 2 hard drive pools used for backup are mounted respectively on `/srv/pool1` and `/srv/pool2`.
- You have a backblaze account.
-First, let's define a variable to hold all our repositories you want to back up to:
+First, let's define a variable to hold all the repositories we want to back up to:
```nix
repos = [
@@ -209,19 +174,38 @@ below) is the former splits the backups into sub-folders on the repositories.
shb.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"];
```
-## Demo {#blocks-backup-demo}
+## Demo {#blocks-restic-demo}
[WIP]
-## Monitoring {#blocks-backup-monitoring}
+## Monitoring {#blocks-restic-monitoring}
[WIP]
-## Maintenance {#blocks-backup-maintenance}
+## Maintenance {#blocks-restic-maintenance}
-[WIP]
+One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
-## Options Reference {#blocks-backup-options}
+In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:
+
+```bash
+restic-myfolder1_srv_pool1_backups
+restic-myfolder1_srv_pool2_backups
+restic-myfolder1_s3_s3.us-west-000.backblazeb2.com_backups
+restic-myfolder2_srv_pool1_backups
+restic-myfolder2_srv_pool2_backups
+restic-myfolder2_s3_s3.us-west-000.backblazeb2.com_backups
+```
+
+Discovering those is easy thanks to tab-completion.
+
+One can then restore a backup with:
+
+```bash
+restic-myfolder1_srv_pool1_backups restore latest -t /
+```
+
+## Options Reference {#blocks-restic-options}
```{=include=} options
id-prefix: blocks-backup-options-
diff --git a/modules/contracts/backup.nix b/modules/contracts/backup.nix
new file mode 100644
index 0000000..0f983d4
--- /dev/null
+++ b/modules/contracts/backup.nix
@@ -0,0 +1,61 @@
+{ lib, ... }:
+lib.types.submodule {
+ freeformType = lib.types.anything;
+
+ options = {
+ user = lib.mkOption {
+ description = "Unix user doing the backups.";
+ type = lib.types.str;
+ default = "backup";
+ };
+
+ group = lib.mkOption {
+ description = "Unix group doing the backups.";
+ type = lib.types.str;
+ default = "backup";
+ };
+
+ sourceDirectories = lib.mkOption {
+ description = "Directories to backup.";
+ type = lib.types.nonEmptyListOf lib.types.str;
+ };
+
+ excludePatterns = lib.mkOption {
+ description = "Patterns to exclude.";
+ type = lib.types.listOf lib.types.str;
+ default = [];
+ };
+
+ retention = lib.mkOption {
+ description = "Backup files retention.";
+ type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]);
+ default = {
+ keep_within = "1d";
+ keep_hourly = 24;
+ keep_daily = 7;
+ keep_weekly = 4;
+ keep_monthly = 6;
+ };
+ };
+
+ hooks = lib.mkOption {
+ description = "Hooks to run around the backup.";
+ default = {};
+ type = lib.types.submodule {
+ options = {
+ before_backup = lib.mkOption {
+ description = "Hooks to run before backup";
+ type = lib.types.listOf lib.types.str;
+ default = [];
+ };
+
+ after_backup = lib.mkOption {
+ description = "Hooks to run after backup";
+ type = lib.types.listOf lib.types.str;
+ default = [];
+ };
+ };
+ };
+ };
+ };
+}
diff --git a/modules/contracts/backup/docs/default.md b/modules/contracts/backup/docs/default.md
new file mode 100644
index 0000000..4e8d790
--- /dev/null
+++ b/modules/contracts/backup/docs/default.md
@@ -0,0 +1,56 @@
+# Backup Contract {#backup-contract}
+
+This NixOS contract represents a backup job
+that will backup one or more files or directories
+at a regular schedule.
+
+## Contract Reference {#backup-contract-options}
+
+These are all the options that are expected to exist for this contract to be respected.
+
+```{=include=} options
+id-prefix: contracts-backup-options-
+list-id: selfhostblocks-options
+source: @OPTIONS_JSON@
+```
+
+## Usage {#backup-contract-usage}
+
+A service that can be backed up will provide a `backup` option, like for the [Vaultwarden service][vaultwarden-service-backup].
+What this option defines is an implementation detail of that service
+but it will at least define what directories to backup
+and possibly hooks to run before or after the backup job runs.
+
+[vaultwarden-service-backup]: services-vaultwarden.html#services-vaultwarden-options-shb.vaultwarden.backup
+
+```nix
+shb.<service>.backup
+```
+
+Let's assume a module implementing this contract is available under the `shb.<backup_impl>` variable.
+Then, to actually backup the service, one would write:
+
+```nix
+shb.<backup_impl>.instances."<service>" = shb.<service>.backup // {
+ enable = true;
+
+ // Options specific to backup_impl
+};
+```
+
+Then, for extra caution, a second backup could be made using another module `shb.<backup_impl_2>`:
+
+```nix
+shb.<backup_impl_2>.instances."<service>" = shb.<service>.backup // {
+ enable = true;
+
+ // Options specific to backup_impl_2
+};
+```
+
+## Provided Implementations {#backup-contract-impl}
+
+One implementation is provided out of the box:
+- [Restic block](blocks-restic.html).
+
+A second one based on `borgbackup` is in progress.
diff --git a/modules/contracts/backup/dummyModule.nix b/modules/contracts/backup/dummyModule.nix
new file mode 100644
index 0000000..8a7d3a8
--- /dev/null
+++ b/modules/contracts/backup/dummyModule.nix
@@ -0,0 +1,10 @@
+{ pkgs, lib, ... }:
+let
+ contracts = pkgs.callPackage ../. {};
+in
+{
+ options.shb.contracts.backup = lib.mkOption {
+ description = "Contract for backups.";
+ type = contracts.backup;
+ };
+}
diff --git a/modules/contracts/default.nix b/modules/contracts/default.nix
index 54a06f6..e748008 100644
--- a/modules/contracts/default.nix
+++ b/modules/contracts/default.nix
@@ -1,5 +1,6 @@
{ lib }:
{
+ backup = import ./backup.nix { inherit lib; };
mount = import ./mount.nix { inherit lib; };
ssl = import ./ssl.nix { inherit lib; };
}
diff --git a/modules/services/vaultwarden.nix b/modules/services/vaultwarden.nix
index ddfdfa0..21ee588 100644
--- a/modules/services/vaultwarden.nix
+++ b/modules/services/vaultwarden.nix
@@ -114,10 +114,26 @@ in
default = { path = dataFolder; };
};
- backupConfig = lib.mkOption {
- type = lib.types.nullOr lib.types.anything;
- description = "Backup configuration of Vaultwarden.";
- default = null;
+ backup = lib.mkOption {
+ type = contracts.backup;
+ description = ''
+ Backup configuration. This is an output option.
+
+ Use it to initialize a block implementing the "backup" contract.
+ For example, with the restic block:
+
+ ```
+ shb.restic.instances."vaultwarden" = {
+ poolName = "root";
+ } // config.shb.vaultwarden.backup;
+ ```
+ '';
+ readOnly = true;
+ default = {
+ sourceDirectories = [
+ dataFolder
+ ];
+ };
};
debug = lib.mkOption {
@@ -217,14 +233,6 @@ in
members = [ "backup" ];
};
- shb.backup.instances.vaultwarden = lib.mkIf (cfg.backupConfig != null) (
- cfg.backupConfig //
- {
- sourceDirectories = [
- config.services.vaultwarden.config.DATA_FOLDER
- ];
- });
-
# TODO: make this work.
# It does not work because it leads to infinite recursion.
# ${cfg.mount}.path = dataFolder;
diff --git a/modules/services/vaultwarden/docs/default.md b/modules/services/vaultwarden/docs/default.md
index 5886f7c..af1a6d1 100644
--- a/modules/services/vaultwarden/docs/default.md
+++ b/modules/services/vaultwarden/docs/default.md
@@ -91,3 +91,25 @@ Integration with the ZFS block allows to automatically create the relevant datas
shb.zfs.datasets."vaultwarden" = config.shb.vaultwarden.mount;
shb.zfs.datasets."postgresql".path = "/var/lib/postgresql";
```
+
+## Maintenance {#services-vaultwarden-maintenance}
+
+No command-line tool is provided to administer Vaultwarden.
+
+Instead, the admin section can be found at the `/admin` endpoint.
+
+## Debug {#services-backup-debug}
+
+In case of an issue, check the logs of the `vaultwarden.service` systemd service.
+
+Enable verbose logging by setting the `shb.vaultwarden.debug` boolean to `true`.
+
+Access the database with `sudo -u vaultwarden psql`.
+
+## Options Reference {#services-vaultwarden-options}
+
+```{=include=} options
+id-prefix: services-vaultwarden-options-
+list-id: selfhostblocks-vaultwarden-options
+source: @OPTIONS_JSON@
+```
diff --git a/test/blocks/restic.nix b/test/blocks/restic.nix
new file mode 100644
index 0000000..12863c5
--- /dev/null
+++ b/test/blocks/restic.nix
@@ -0,0 +1,156 @@
+{ pkgs, lib, ... }:
+let
+ pkgs' = pkgs;
+
+ testLib = pkgs.callPackage ../common.nix {};
+
+ base = testLib.base [
+ ../../modules/blocks/restic.nix
+ ];
+in
+{
+ backupAndRestore = pkgs.testers.runNixOSTest {
+ name = "restic_backupAndRestore";
+
+ nodes.machine = {
+ imports = ( testLib.baseImports pkgs' ) ++ [
+ ../../modules/blocks/restic.nix
+ ];
+
+ shb.restic = {
+ user = "root";
+ group = "root";
+ };
+ shb.restic.instances."testinstance" = {
+ enable = true;
+
+ passphraseFile = pkgs.writeText "passphrase" "PassPhrase";
+
+ sourceDirectories = [
+ "/opt/files/A"
+ "/opt/files/B"
+ ];
+
+ repositories = [
+ {
+ path = "/opt/repos/A";
+ timerConfig = {
+ OnCalendar = "00:00:00";
+ RandomizedDelaySec = "5h";
+ };
+ }
+ {
+ path = "/opt/repos/B";
+ timerConfig = {
+ OnCalendar = "00:00:00";
+ RandomizedDelaySec = "5h";
+ };
+ }
+ ];
+ };
+ };
+
+ extraPythonPackages = p: [ p.dictdiffer ];
+ skipTypeCheck = true;
+
+ testScript = { nodes, ... }: let
+ instanceCfg = nodes.machine.shb.restic.instances."testinstance";
+ in ''
+ from dictdiffer import diff
+
+ def list_files(dir):
+ files_and_content = {}
+
+ files = machine.succeed(f"""
+ find {dir} -type f
+ """).split("\n")[:-1]
+
+ for f in files:
+ content = machine.succeed(f"""
+ cat {f}
+ """).strip()
+ files_and_content[f] = content
+
+ return files_and_content
+
+ def assert_files(dir, files):
+ result = list(diff(list_files(dir), files))
+ if len(result) > 0:
+ raise Exception("Unexpected files:", result)
+
+ with subtest("Create initial content"):
+ machine.succeed("""
+ mkdir -p /opt/files/A
+ mkdir -p /opt/files/B
+ mkdir -p /opt/repos/A
+ mkdir -p /opt/repos/B
+
+ echo repoA_fileA_1 > /opt/files/A/fileA
+ echo repoA_fileB_1 > /opt/files/A/fileB
+ echo repoB_fileA_1 > /opt/files/B/fileA
+ echo repoB_fileB_1 > /opt/files/B/fileB
+
+ # chown :backup -R /opt/files
+ """)
+
+ assert_files("/opt/files", {
+ '/opt/files/B/fileA': 'repoB_fileA_1',
+ '/opt/files/B/fileB': 'repoB_fileB_1',
+ '/opt/files/A/fileA': 'repoA_fileA_1',
+ '/opt/files/A/fileB': 'repoA_fileB_1',
+ })
+
+ with subtest("First backup in repo A"):
+ machine.succeed("systemctl start restic-backups-testinstance_opt_repos_A")
+
+ with subtest("New content"):
+ machine.succeed("""
+ echo repoA_fileA_2 > /opt/files/A/fileA
+ echo repoA_fileB_2 > /opt/files/A/fileB
+ echo repoB_fileA_2 > /opt/files/B/fileA
+ echo repoB_fileB_2 > /opt/files/B/fileB
+ """)
+
+ assert_files("/opt/files", {
+ '/opt/files/B/fileA': 'repoB_fileA_2',
+ '/opt/files/B/fileB': 'repoB_fileB_2',
+ '/opt/files/A/fileA': 'repoA_fileA_2',
+ '/opt/files/A/fileB': 'repoA_fileB_2',
+ })
+
+ with subtest("Second backup in repo B"):
+ machine.succeed("systemctl start restic-backups-testinstance_opt_repos_B")
+
+ with subtest("Delete content"):
+ machine.succeed("""
+ rm -r /opt/files/A /opt/files/B
+ """)
+
+ assert_files("/opt/files", {})
+
+ with subtest("Restore initial content from repo A"):
+ machine.succeed("""
+ restic-testinstance_opt_repos_A restore latest -t /
+ """)
+
+ assert_files("/opt/files", {
+ '/opt/files/B/fileA': 'repoB_fileA_1',
+ '/opt/files/B/fileB': 'repoB_fileB_1',
+ '/opt/files/A/fileA': 'repoA_fileA_1',
+ '/opt/files/A/fileB': 'repoA_fileB_1',
+ })
+
+ with subtest("Restore initial content from repo B"):
+ machine.succeed("""
+ restic-testinstance_opt_repos_B restore latest -t /
+ """)
+
+ assert_files("/opt/files", {
+ '/opt/files/B/fileA': 'repoB_fileA_2',
+ '/opt/files/B/fileB': 'repoB_fileB_2',
+ '/opt/files/A/fileA': 'repoA_fileA_2',
+ '/opt/files/A/fileB': 'repoA_fileB_2',
+ })
+ '';
+ };
+}
diff --git a/test/common.nix b/test/common.nix
index abeef15..88a1322 100644
--- a/test/common.nix
+++ b/test/common.nix
@@ -1,6 +1,12 @@
{
lib,
}:
+let
+ baseImports = pkgs: [
+ (pkgs.path + "/nixos/modules/profiles/headless.nix")
+ (pkgs.path + "/nixos/modules/profiles/qemu-guest.nix")
+ ];
+in
{
accessScript = {
subdomain
@@ -92,21 +98,24 @@
${indent 4 script}
'');
+ inherit baseImports;
+
base = pkgs: additionalModules: {
- imports = [
- (pkgs.path + "/nixos/modules/profiles/headless.nix")
- (pkgs.path + "/nixos/modules/profiles/qemu-guest.nix")
- # TODO: replace this option by the backup contract
- {
- options = {
- shb.backup = lib.mkOption { type = lib.types.anything; };
- };
- }
- # TODO: replace postgresql.nix and authelia.nix by the sso contract
- ../modules/blocks/postgresql.nix
- ../modules/blocks/authelia.nix
- ../modules/blocks/nginx.nix
- ] ++ additionalModules;
+ imports =
+ ( baseImports pkgs )
+ ++ [
+ # TODO: replace this option by the backup contract
+ {
+ options = {
+ shb.backup = lib.mkOption { type = lib.types.anything; };
+ };
+ }
+ # TODO: replace postgresql.nix and authelia.nix by the sso contract
+ ../modules/blocks/postgresql.nix
+ ../modules/blocks/authelia.nix
+ ../modules/blocks/nginx.nix
+ ]
+ ++ additionalModules;
# Nginx port.
networking.firewall.allowedTCPPorts = [ 80 443 ];