From 7c31bc5bda3a341879bc787d07a5c2806ed1aa36 Mon Sep 17 00:00:00 2001
From: ibizaman <ibizapeanut@gmail.com>
Date: Thu, 22 Aug 2024 03:39:24 +0200
Subject: [PATCH] fix restic secrets
---
lib/default.nix | 45 ++++++++++++++++++++-
modules/blocks/restic.nix | 58 +++++++++++++++++++--------
modules/blocks/restic/docs/default.md | 4 +-
test/blocks/restic.nix | 24 +++++++++++
4 files changed, 111 insertions(+), 20 deletions(-)
diff --git a/lib/default.nix b/lib/default.nix
index dde07f6..2b8ccd1 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,4 +1,8 @@
{ pkgs, lib }:
+let
+ inherit (builtins) isAttrs hasAttr;
+ inherit (lib) concatStringsSep;
+in
rec {
# Replace secrets in a file.
# - userConfig is an attrset that will produce a config file.
@@ -228,8 +232,47 @@ rec {
results = pkgs.lib.runTests tests;
in
if results != [ ] then
- builtins.throw (builtins.concatStringsSep "\n" (map resultToString results))
+ builtins.throw (builtins.concatStringsSep "\n" (map resultToString (lib.traceValSeqN 3 results)))
else
pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
+
+ genConfigOutOfBandSystemd = { config, configLocation, generator }:
+ {
+ loadCredentials = getLoadCredentials "source" config;
+ preStart = lib.mkBefore (replaceSecrets {
+ userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
+ resultPath = configLocation;
+ inherit generator;
+ });
+ };
+
+ updateToLoadCredentials = sourceField: rootDir: attrs:
+ let
+ hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+
+ valueOrLoadCredential = path: value:
+ if ! (hasPlaceholderField value)
+ then value
+ else value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
+ in
+ mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) valueOrLoadCredential attrs;
+
+ getLoadCredentials = sourceField: attrs:
+ let
+ hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+
+ addPathField = path: value:
+ if ! (hasPlaceholderField value)
+ then value
+ else value // { inherit path; };
+
+ secretsWithPath = mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) addPathField attrs;
+
+ allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
+
+ genLoadCredentials = secret:
+ "${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
+ in
+ map genLoadCredentials allSecrets;
}
diff --git a/modules/blocks/restic.nix b/modules/blocks/restic.nix
index 1eb82d4..849925d 100644
--- a/modules/blocks/restic.nix
+++ b/modules/blocks/restic.nix
@@ -20,7 +20,6 @@ let
For Restic, the same user must be used for all instances.
'';
type = lib.types.str;
- readOnly = true;
default = cfg.user;
};
@@ -31,7 +30,6 @@ let
For Restic, the same group must be used for all instances.
'';
type = lib.types.str;
- readOnly = true;
default = cfg.group;
};
@@ -55,13 +53,15 @@ let
description = "Repository location";
};
- extraSecrets = lib.mkOption {
+ secrets = lib.mkOption {
type = lib.types.attrsOf shblib.secretFileType;
default = {};
description = ''
- Extra secrets needed to access the repository where the backups will be stored.
+ Secrets needed to access the repository where the backups will be stored.
+
+ See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
+ and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
- See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example.
'';
example = lib.literalExpression ''
{
@@ -228,12 +228,6 @@ in
backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
- } // lib.attrsets.optionalAttrs (repository.extraSecrets != {}) {
- environmentFile = shblib.replaceSecrets {
- userConfig = repository.extraSecrets;
- resultPath = "/var/lib/backup/${name}";
- generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} v);
- };
} // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
exclude = instance.excludePatterns;
};
@@ -245,12 +239,42 @@ in
systemd.services =
let
- mkRepositorySettings = name: instance: repository: {
- "restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = {
- Nice = cfg.performance.niceness;
- IOSchedulingClass = cfg.performance.ioSchedulingClass;
- IOSchedulingPriority = cfg.performance.ioPriority;
- };
+ mkRepositorySettings = name: instance: repository:
+ let
+ serviceName = "restic-backups-${name}_${repoSlugName repository.path}";
+ in
+ {
+ ${serviceName} = lib.mkMerge [
+ {
+ serviceConfig = {
+ Nice = cfg.performance.niceness;
+ IOSchedulingClass = cfg.performance.ioSchedulingClass;
+ IOSchedulingPriority = cfg.performance.ioPriority;
+ };
+ }
+ (lib.attrsets.optionalAttrs (repository.secrets != {})
+ {
+ serviceConfig.EnvironmentFile = [
+ "/run/secrets/restic/${serviceName}"
+ ];
+ after = [ "${serviceName}-pre.service" ];
+ requires = [ "${serviceName}-pre.service" ];
+ })
+ ];
+
+ "${serviceName}-pre" = lib.mkIf (repository.secrets != {})
+ (let
+ script = shblib.genConfigOutOfBandSystemd {
+ config = repository.secrets;
+ configLocation = "/run/secrets/restic/${serviceName}";
+ generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} { globalSection = v; });
+ };
+ in
+ {
+ script = script.preStart;
+ serviceConfig.Type = "oneshot";
+ serviceConfig.LoadCredential = script.loadCredentials;
+ });
};
mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
in
diff --git a/modules/blocks/restic/docs/default.md b/modules/blocks/restic/docs/default.md
index 7e09165..477aa7c 100644
--- a/modules/blocks/restic/docs/default.md
+++ b/modules/blocks/restic/docs/default.md
@@ -76,8 +76,8 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
};
+ extraSecrets = {
-+ AWS_ACCESS_KEY_ID="<path/to/access_key_id>";
-+ AWS_SECRET_ACCESS_KEY="<path/to/secret_access_key>";
++ AWS_ACCESS_KEY_ID.source="<path/to/access_key_id>";
++ AWS_SECRET_ACCESS_KEY.source="<path/to/secret_access_key>";
+ };
}];
}
diff --git a/test/blocks/restic.nix b/test/blocks/restic.nix
index 12863c5..79ab5e5 100644
--- a/test/blocks/restic.nix
+++ b/test/blocks/restic.nix
@@ -3,6 +3,7 @@ let
pkgs' = pkgs;
testLib = pkgs.callPackage ../common.nix {};
+ shblib = pkgs.callPackage ../../lib {};
base = testLib.base [
../../modules/blocks/restic.nix
@@ -38,6 +39,12 @@ in
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
+ # Those are not needed by the repository but are still included
+ # so we can test them in the hooks section.
+ secrets = {
+ A.source = pkgs.writeText "A" "secretA";
+ B.source = pkgs.writeText "B" "secretB";
+ };
}
{
path = "/opt/repos/B";
@@ -47,6 +54,23 @@ in
};
}
];
+
+ hooks.before_backup = [''
+ echo $RUNTIME_DIRECTORY
+ if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
+ if ! [ -f /run/secrets/restic/restic-backups-testinstance_opt_repos_A ]; then
+ exit 10
+ fi
+ if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
+ echo "A:$A"
+ exit 11
+ fi
+ if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
+ echo "A:$A"
+ exit 12
+ fi
+ fi
+ ''];
};
};