fix restic secrets

lib/default.nix

@@ -1,4 +1,8 @@
 { pkgs, lib }:
+let
+  inherit (builtins) isAttrs hasAttr;
+  inherit (lib) concatStringsSep;
+in
 rec {
   # Replace secrets in a file.
   # - userConfig is an attrset that will produce a config file.
@@ -228,8 +232,47 @@ rec {
       results = pkgs.lib.runTests tests;
     in
     if results != [ ] then
-      builtins.throw (builtins.concatStringsSep "\n" (map resultToString results))
+      builtins.throw (builtins.concatStringsSep "\n" (map resultToString (lib.traceValSeqN 3 results)))
     else
       pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
 
+
+  genConfigOutOfBandSystemd = { config, configLocation, generator }:
+    {
+      loadCredentials = getLoadCredentials "source" config;
+      preStart = lib.mkBefore (replaceSecrets {
+        userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
+        resultPath = configLocation;
+        inherit generator;
+      });
+    };
+
+  updateToLoadCredentials = sourceField: rootDir: attrs:
+    let
+      hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+
+      valueOrLoadCredential = path: value:
+        if ! (hasPlaceholderField value)
+        then value
+        else value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
+    in
+      mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) valueOrLoadCredential attrs;
+
+  getLoadCredentials = sourceField: attrs:
+    let
+      hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+
+      addPathField = path: value:
+        if ! (hasPlaceholderField value)
+        then value
+        else value // { inherit path; };
+
+      secretsWithPath = mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) addPathField attrs;
+
+      allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
+
+      genLoadCredentials = secret:
+        "${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
+    in
+      map genLoadCredentials allSecrets;
 }

modules/blocks/restic.nix

@@ -20,7 +20,6 @@ let
         For Restic, the same user must be used for all instances.
       '';
       type = lib.types.str;
-      readOnly = true;
       default = cfg.user;
     };
 
@@ -31,7 +30,6 @@ let
         For Restic, the same group must be used for all instances.
       '';
       type = lib.types.str;
-      readOnly = true;
       default = cfg.group;
     };
 
@@ -55,13 +53,15 @@ let
             description = "Repository location";
           };
 
-          extraSecrets = lib.mkOption {
+          secrets = lib.mkOption {
             type = lib.types.attrsOf shblib.secretFileType;
             default = {};
             description = ''
-              Extra secrets needed to access the repository where the backups will be stored.
+              Secrets needed to access the repository where the backups will be stored.
+
+              See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
+              and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
 
-              See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example.
               '';
             example = lib.literalExpression ''
               {
@@ -228,12 +228,6 @@ in
                 backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
 
                 backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
-              } // lib.attrsets.optionalAttrs (repository.extraSecrets != {}) {
-                environmentFile = shblib.replaceSecrets {
-                  userConfig = repository.extraSecrets;
-                  resultPath = "/var/lib/backup/${name}";
-                  generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} v);
-                };
               } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
                 exclude = instance.excludePatterns;
               };
@@ -245,12 +239,42 @@ in
 
         systemd.services =
           let
-            mkRepositorySettings = name: instance: repository: {
-              "restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = {
-                Nice = cfg.performance.niceness;
-                IOSchedulingClass = cfg.performance.ioSchedulingClass;
-                IOSchedulingPriority = cfg.performance.ioPriority;
-              };
+            mkRepositorySettings = name: instance: repository:
+              let
+                serviceName = "restic-backups-${name}_${repoSlugName repository.path}";
+              in
+                {
+                  ${serviceName} = lib.mkMerge [
+                    {
+                      serviceConfig = {
+                        Nice = cfg.performance.niceness;
+                        IOSchedulingClass = cfg.performance.ioSchedulingClass;
+                        IOSchedulingPriority = cfg.performance.ioPriority;
+                      };
+                    }
+                    (lib.attrsets.optionalAttrs (repository.secrets != {})
+                      {
+                        serviceConfig.EnvironmentFile = [
+                          "/run/secrets/restic/${serviceName}"
+                        ];
+                        after = [ "${serviceName}-pre.service" ];
+                        requires = [ "${serviceName}-pre.service" ];
+                      })
+                  ];
+
+                  "${serviceName}-pre" = lib.mkIf (repository.secrets != {})
+                    (let
+                      script = shblib.genConfigOutOfBandSystemd {
+                        config = repository.secrets;
+                        configLocation = "/run/secrets/restic/${serviceName}";
+                        generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} { globalSection = v; });
+                      };
+                    in
+                      {
+                        script = script.preStart;
+                        serviceConfig.Type = "oneshot";
+                        serviceConfig.LoadCredential = script.loadCredentials;
+                      });
             };
             mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
           in

modules/blocks/restic/docs/default.md

@@ -76,8 +76,8 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
       };
 
 +     extraSecrets = {
-+       AWS_ACCESS_KEY_ID="<path/to/access_key_id>";
-+       AWS_SECRET_ACCESS_KEY="<path/to/secret_access_key>";
++       AWS_ACCESS_KEY_ID.source="<path/to/access_key_id>";
++       AWS_SECRET_ACCESS_KEY.source="<path/to/secret_access_key>";
 +     };
     }];
   }

test/blocks/restic.nix

@@ -3,6 +3,7 @@ let
   pkgs' = pkgs;
 
   testLib = pkgs.callPackage ../common.nix {};
+  shblib = pkgs.callPackage ../../lib {};
 
   base = testLib.base [
     ../../modules/blocks/restic.nix
@@ -38,6 +39,12 @@ in
               OnCalendar = "00:00:00";
               RandomizedDelaySec = "5h";
             };
+            # Those are not needed by the repository but are still included
+            # so we can test them in the hooks section.
+            secrets = {
+              A.source = pkgs.writeText "A" "secretA";
+              B.source = pkgs.writeText "B" "secretB";
+            };
           }
           {
             path = "/opt/repos/B";
@@ -47,6 +54,23 @@ in
             };
           }
         ];
+
+        hooks.before_backup = [''
+        echo $RUNTIME_DIRECTORY
+        if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
+          if ! [ -f /run/secrets/restic/restic-backups-testinstance_opt_repos_A ]; then
+            exit 10
+          fi
+          if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
+            echo "A:$A"
+            exit 11
+          fi
+          if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
+            echo "A:$A"
+            exit 12
+          fi
+        fi
+        ''];
       };
     };