Fix backup contract secrets (#280)

lib/default.nix

@@ -1,4 +1,8 @@
 { pkgs, lib }:
+let
+  inherit (builtins) isAttrs hasAttr;
+  inherit (lib) concatStringsSep;
+in
 rec {
   # Replace secrets in a file.
   # - userConfig is an attrset that will produce a config file.
@@ -228,8 +232,47 @@ rec {
       results = pkgs.lib.runTests tests;
     in
     if results != [ ] then
-      builtins.throw (builtins.concatStringsSep "\n" (map resultToString results))
+      builtins.throw (builtins.concatStringsSep "\n" (map resultToString (lib.traceValSeqN 3 results)))
     else
       pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
 
+
+  genConfigOutOfBandSystemd = { config, configLocation, generator }:
+    {
+      loadCredentials = getLoadCredentials "source" config;
+      preStart = lib.mkBefore (replaceSecrets {
+        userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
+        resultPath = configLocation;
+        inherit generator;
+      });
+    };
+
+  updateToLoadCredentials = sourceField: rootDir: attrs:
+    let
+      hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+
+      valueOrLoadCredential = path: value:
+        if ! (hasPlaceholderField value)
+        then value
+        else value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
+    in
+      mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) valueOrLoadCredential attrs;
+
+  getLoadCredentials = sourceField: attrs:
+    let
+      hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+
+      addPathField = path: value:
+        if ! (hasPlaceholderField value)
+        then value
+        else value // { inherit path; };
+
+      secretsWithPath = mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) addPathField attrs;
+
+      allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
+
+      genLoadCredentials = secret:
+        "${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
+    in
+      map genLoadCredentials allSecrets;
 }

modules/blocks/monitoring.nix

@@ -1,4 +1,4 @@
-{ config, options, pkgs, lib, ... }:
+{ config, pkgs, lib, ... }:
 
 let
   cfg = config.shb.monitoring;

modules/blocks/restic.nix

@@ -13,6 +13,26 @@ let
       type = lib.types.path;
     };
 
+    user = lib.mkOption {
+      description = ''
+        Unix user doing the backups.
+
+        For Restic, the same user must be used for all instances.
+      '';
+      type = lib.types.str;
+      default = cfg.user;
+    };
+
+    group = lib.mkOption {
+      description = ''
+        Unix group doing the backups.
+
+        For Restic, the same group must be used for all instances.
+      '';
+      type = lib.types.str;
+      default = cfg.group;
+    };
+
     sourceDirectories = lib.mkOption {
       description = "Source directories.";
       type = lib.types.nonEmptyListOf lib.types.str;
@@ -33,13 +53,15 @@ let
             description = "Repository location";
           };
 
-          extraSecrets = lib.mkOption {
+          secrets = lib.mkOption {
             type = lib.types.attrsOf shblib.secretFileType;
             default = {};
             description = ''
-              Extra secrets needed to access the repository where the backups will be stored.
+              Secrets needed to access the repository where the backups will be stored.
+
+              See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
+              and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
 
-              See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example.
               '';
             example = lib.literalExpression ''
               {
@@ -154,6 +176,17 @@ in
       enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
     in lib.mkMerge [
       {
+        assertions = [
+          {
+            assertion = lib.all (x: x.user == cfg.user) (lib.mapAttrsToList (n: v: v)cfg.instances);
+            message = "All Restic instances must have the same user as 'shb.restic.user'.";
+          }
+          {
+            assertion = lib.all (x: x.group == cfg.group) (lib.mapAttrsToList (n: v: v) cfg.instances);
+            message = "All Restic instances must have the same group as 'shb.restic.group'.";
+          }
+        ];
+
         users.users = {
           ${cfg.user} = {
             name = cfg.user;
@@ -195,12 +228,6 @@ in
                 backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
 
                 backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
-              } // lib.attrsets.optionalAttrs (repository.extraSecrets != {}) {
-                environmentFile = shblib.replaceSecrets {
-                  userConfig = repository.extraSecrets;
-                  resultPath = "/var/lib/backup/${name}";
-                  generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} v);
-                };
               } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
                 exclude = instance.excludePatterns;
               };
@@ -212,12 +239,42 @@ in
 
         systemd.services =
           let
-            mkRepositorySettings = name: instance: repository: {
+            mkRepositorySettings = name: instance: repository:
-              "restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = {
+              let
+                serviceName = "restic-backups-${name}_${repoSlugName repository.path}";
+              in
+                {
+                  ${serviceName} = lib.mkMerge [
+                    {
+                      serviceConfig = {
                         Nice = cfg.performance.niceness;
                         IOSchedulingClass = cfg.performance.ioSchedulingClass;
                         IOSchedulingPriority = cfg.performance.ioPriority;
                       };
+                    }
+                    (lib.attrsets.optionalAttrs (repository.secrets != {})
+                      {
+                        serviceConfig.EnvironmentFile = [
+                          "/run/secrets/restic/${serviceName}"
+                        ];
+                        after = [ "${serviceName}-pre.service" ];
+                        requires = [ "${serviceName}-pre.service" ];
+                      })
+                  ];
+
+                  "${serviceName}-pre" = lib.mkIf (repository.secrets != {})
+                    (let
+                      script = shblib.genConfigOutOfBandSystemd {
+                        config = repository.secrets;
+                        configLocation = "/run/secrets/restic/${serviceName}";
+                        generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} { globalSection = v; });
+                      };
+                    in
+                      {
+                        script = script.preStart;
+                        serviceConfig.Type = "oneshot";
+                        serviceConfig.LoadCredential = script.loadCredentials;
+                      });
             };
             mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
           in

modules/blocks/restic/docs/default.md

@@ -76,8 +76,8 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
       };
 
 +     extraSecrets = {
-+       AWS_ACCESS_KEY_ID="<path/to/access_key_id>";
++       AWS_ACCESS_KEY_ID.source="<path/to/access_key_id>";
-+       AWS_SECRET_ACCESS_KEY="<path/to/secret_access_key>";
++       AWS_SECRET_ACCESS_KEY.source="<path/to/secret_access_key>";
 +     };
     }];
   }

modules/services/arr.nix

@@ -1,4 +1,4 @@
-{ config, options, pkgs, lib, ... }:
+{ config, pkgs, lib, ... }:
 
 let
   cfg = config.shb.arr;

modules/services/nextcloud-server.nix

@@ -1,8 +1,7 @@
-{ config, options, pkgs, lib, ... }:
+{ config, pkgs, lib, ... }:
 
 let
   cfg = config.shb.nextcloud;
-  opt = options.shb.nextcloud;
 
   fqdn = "${cfg.subdomain}.${cfg.domain}";
   fqdnWithPort = if isNull cfg.port then fqdn else "${fqdn}:${toString cfg.port}";

test/blocks/restic.nix

@@ -3,6 +3,7 @@ let
   pkgs' = pkgs;
 
   testLib = pkgs.callPackage ../common.nix {};
+  shblib = pkgs.callPackage ../../lib {};
 
   base = testLib.base [
     ../../modules/blocks/restic.nix
@@ -38,6 +39,12 @@ in
               OnCalendar = "00:00:00";
               RandomizedDelaySec = "5h";
             };
+            # Those are not needed by the repository but are still included
+            # so we can test them in the hooks section.
+            secrets = {
+              A.source = pkgs.writeText "A" "secretA";
+              B.source = pkgs.writeText "B" "secretB";
+            };
           }
           {
             path = "/opt/repos/B";
@@ -47,6 +54,23 @@ in
             };
           }
         ];
+
+        hooks.before_backup = [''
+        echo $RUNTIME_DIRECTORY
+        if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
+          if ! [ -f /run/secrets/restic/restic-backups-testinstance_opt_repos_A ]; then
+            exit 10
+          fi
+          if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
+            echo "A:$A"
+            exit 11
+          fi
+          if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
+            echo "A:$A"
+            exit 12
+          fi
+        fi
+        ''];
       };
     };