move templating code to lib file

flake.nix

@@ -88,13 +88,20 @@
                 mergeTests (importFiles [
                   ./test/modules/arr.nix
                   ./test/modules/davfs.nix
+                  ./test/modules/lib.nix
                   ./test/modules/nginx.nix
                   ./test/modules/postgresql.nix
                 ]);
             };
+
+            lib = nix-flake-tests.lib.check {
+              inherit pkgs;
+              tests = pkgs.callPackage ./test/modules/lib.nix {};
+            };
           }
           // (vm_test "authelia" ./test/vm/authelia.nix)
           // (vm_test "ldap" ./test/vm/ldap.nix)
+          // (vm_test "lib" ./test/vm/lib.nix)
           // (vm_test "postgresql" ./test/vm/postgresql.nix)
           // (vm_test "monitoring" ./test/vm/monitoring.nix)
           // (vm_test "nextcloud" ./test/vm/nextcloud.nix)

lib/default.nix

@@ -1,13 +1,110 @@
-{ lib }:
+{ pkgs, lib }:
-{
+rec {
-  template = file: newPath: replacements:
+  replaceSecrets = { userConfig, resultPath, generator }:
     let
-      templatePath = newPath + ".template";
+      configWithTemplates = withReplacements userConfig;
+
+      nonSecretConfigFile = pkgs.writeText "${resultPath}.template" (generator configWithTemplates);
+
+      replacements = getReplacements userConfig;
+    in
+      replaceSecretsScript {
+        file = nonSecretConfigFile;
+        inherit resultPath replacements;
+      };
+
+  template = file: newPath: replacements: replaceSecretsScript { inherit file replacements; resultPath = newPath; };
+  replaceSecretsScript = { file, resultPath, replacements }:
+    let
+      templatePath = resultPath + ".template";
       sedPatterns = lib.strings.concatStringsSep " " (lib.attrsets.mapAttrsToList (from: to: "-e \"s|${from}|${to}|\"") replacements);
     in
       ''
+      set -euo pipefail
+      set -x
       ln -fs ${file} ${templatePath}
-      rm ${newPath} || :
+      rm -f ${resultPath}
-      sed ${sedPatterns} ${templatePath} > ${newPath}
+      ${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath}
+      ${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath} > ${resultPath}
       '';
+
+  secretFileType = lib.types.submodule {
+    options = {
+      source = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing the value.";
+      };
+
+      transform = lib.mkOption {
+        type = lib.types.raw;
+        description = "An optional function to transform the secret.";
+        default = null;
+        example = lib.literalExpression ''
+        v: "prefix-$${v}-suffix"
+        '';
+      };
+    };
+  };
+
+  secretName = name:
+      "%SECRET${lib.strings.toUpper (lib.strings.concatMapStrings (s: "_" + s) name)}%";
+
+  withReplacements = attrs:
+    let
+      valueOrReplacement = name: value:
+        if !(builtins.isAttrs value && value ? "source")
+        then value
+        else secretName name;
+    in
+      mapAttrsRecursiveCond (v: ! v ? "source") valueOrReplacement attrs;
+
+  getReplacements = attrs:
+    let
+      addNameField = name: value:
+        if !(builtins.isAttrs value && value ? "source")
+        then value
+        else value // { name = name; };
+
+      secretsWithName = mapAttrsRecursiveCond (v: ! v ? "source") addNameField attrs;
+
+      allSecrets = collect (v: builtins.isAttrs v && v ? "source") secretsWithName;
+
+      t = { transform ? null, ... }: if isNull transform then x: x else transform;
+
+      genReplacement = secret:
+        lib.attrsets.nameValuePair (secretName secret.name) ((t secret) "$(cat ${toString secret.source})");
+    in
+      lib.attrsets.listToAttrs (map genReplacement allSecrets);
+      
+  # Inspired lib.attrsets.mapAttrsRecursiveCond but also recurses on lists.
+  mapAttrsRecursiveCond =
+    # A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set.
+    cond:
+    # A function, given a list of attribute names and a value, returns a new value.
+    f:
+    # Attribute set or list to recursively map over.
+    set:
+    let
+      recurse = path: val:
+        if builtins.isAttrs val && cond val
+        then lib.attrsets.mapAttrs (n: v: recurse (path ++ [n]) v) val
+        else if builtins.isList val && cond val
+        then lib.lists.imap0 (i: v: recurse (path ++ [(builtins.toString i)]) v) val
+        else f path val;
+    in recurse [] set;
+
+  # Like lib.attrsets.collect but also recurses on lists.
+  collect =
+  # Given an attribute's value, determine if recursion should stop.
+  pred:
+  # The attribute set to recursively collect.
+  attrs:
+    if pred attrs then
+      [ attrs ]
+    else if builtins.isAttrs attrs then
+      lib.lists.concatMap (collect pred) (lib.attrsets.attrValues attrs)
+    else if builtins.isList attrs then
+      lib.lists.concatMap (collect pred) attrs
+    else
+      [];
 }

modules/blocks/authelia.nix

@@ -94,9 +94,54 @@ in
     };
 
     oidcClients = lib.mkOption {
-      type = lib.types.listOf lib.types.anything;
       description = "OIDC clients";
       default = [];
+      type = lib.types.listOf (lib.types.submodule {
+        freeformType = lib.types.attrsOf lib.types.anything;
+
+        options = {
+          id = lib.mkOption {
+            type = lib.types.str;
+            description = "Unique identifier of the OIDC client.";
+          };
+
+          description = lib.mkOption {
+            type = lib.types.nullOr lib.types.str;
+            description = "Human readable description of the OIDC client.";
+            default = null;
+          };
+
+          secret = lib.mkOption {
+            type = shblib.secretFileType;
+            description = "File containing the shared secret with the OIDC client.";
+          };
+
+          public = lib.mkOption {
+            type = lib.types.bool;
+            description = "If the OIDC client is public or not.";
+            default = false;
+            apply = v: if v then "true" else "false";
+          };
+
+          authorization_policy = lib.mkOption {
+            type = lib.types.enum [ "one_factor" "two_factor" ];
+            description = "Require one factor (password) or two factor (device) authentication.";
+            default = "one_factor";
+          };
+
+          redirect_uris = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            description = "List of uris that are allowed to be redirected to.";
+          };
+
+          scopes = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            description = "Scopes to ask for";
+            example = [ "openid" "profile" "email" "groups" ];
+            default = [];
+          };
+        };
+      });
     };
 
     smtp = lib.mkOption {
@@ -291,13 +336,13 @@ in
     systemd.services."authelia-${fqdn}".preStart =
       let
         mkCfg = clients:
-        let
+          shblib.replaceSecrets {
-          addTemplate = client: (builtins.removeAttrs client ["secretFile"]) // {secret = "%SECRET_${client.id}%";};
+            userConfig = {
-          tmplFile = pkgs.writeText "oidc_clients.yaml" (lib.generators.toYAML {} {identity_providers.oidc.clients = map addTemplate clients;});
+              identity_providers.oidc.clients = clients;
-          replace = client: {"%SECRET_${client.id}%" = "$(cat ${toString client.secretFile})";};
+            };
-          replacements = lib.foldl (container: client: container // (replace client) ) {} clients;
+            resultPath = "/var/lib/authelia-${fqdn}/oidc_clients.yaml";
-        in
+            generator = lib.generators.toYAML {};
-          shblib.template tmplFile "/var/lib/authelia-${fqdn}/oidc_clients.yaml" replacements;
+          };
       in
         lib.mkBefore (mkCfg cfg.oidcClients);
 

modules/services/home-assistant.nix

@@ -4,6 +4,7 @@ let
   cfg = config.shb.home-assistant;
 
   contracts = pkgs.callPackage ../contracts {};
+  shblib = pkgs.callPackage ../../lib {};
 
   fqdn = "${cfg.subdomain}.${cfg.domain}";
 
@@ -18,6 +19,15 @@ let
     export PATH=${pkgs.gnused}/bin:${pkgs.curl}/bin:${pkgs.jq}/bin
     exec ${pkgs.bash}/bin/bash ${ldap_auth_script_repo}/example_configs/lldap-ha-auth.sh $@
   '';
+
+  # Filter secrets from config. Secrets are those of the form { source = <path>; }
+  secrets = lib.attrsets.filterAttrs (k: v: builtins.isAttrs v) cfg.config;
+
+  nonSecrets = (lib.attrsets.filterAttrs (k: v: !(builtins.isAttrs v)) cfg.config);
+
+  configWithSecretsIncludes =
+    nonSecrets
+    // (lib.attrsets.mapAttrs (k: v: "!secret ${k}") secrets);
 in
 {
   options.shb.home-assistant = {
@@ -41,6 +51,41 @@ in
       default = null;
     };
 
+    config = lib.mkOption {
+      description = "See all available settings at https://www.home-assistant.io/docs/configuration/basic/";
+      type = lib.types.submodule {
+        freeformType = lib.types.attrsOf lib.types.str;
+        options = {
+          name = lib.mkOption {
+            type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
+            description = "Name of the Home Assistant instance.";
+          };
+          country = lib.mkOption {
+            type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
+            description = "Two letter country code where this instance is located.";
+          };
+          latitude = lib.mkOption {
+            type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
+            description = "Latitude where this instance is located.";
+          };
+          longitude = lib.mkOption {
+            type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
+            description = "Longitude where this instance is located.";
+          };
+          time_zone = lib.mkOption {
+            type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
+            description = "Timezone of this instance.";
+            example = "America/Los_Angeles";
+          };
+          unit_system = lib.mkOption {
+            type = lib.types.oneOf [ lib.types.str (lib.types.enum [ "metric" "us_customary" ]) ];
+            description = "Timezone of this instance.";
+            example = "America/Los_Angeles";
+          };
+        };
+      };
+    };
+
     ldap = lib.mkOption {
       description = ''
         LDAP Integration App. [Manual](https://docs.nextcloud.com/server/latest/admin_manual/configuration_user/user_auth_ldap.html)
@@ -91,12 +136,6 @@ in
       };
     };
 
-    sopsFile = lib.mkOption {
-      type = lib.types.path;
-      description = "Sops file location";
-      example = "secrets/homeassistant.yaml";
-    };
-
     backupCfg = lib.mkOption {
       type = lib.types.anything;
       description = "Backup configuration for home-assistant";
@@ -144,14 +183,8 @@ in
           trusted_proxies = "127.0.0.1";
         };
         logger.default = "info";
-        homeassistant = {
+        homeassistant = configWithSecretsIncludes // {
           external_url = "https://${cfg.subdomain}.${cfg.domain}";
-          name = "!secret name";
-          country = "!secret country";
-          latitude = "!secret latitude_home";
-          longitude = "!secret longitude_home";
-          time_zone = "!secret time_zone";
-          unit_system = "metric";
           auth_providers =
             (lib.optionals (!cfg.ldap.enable || cfg.ldap.keepDefaultAuth) [
               {
@@ -256,23 +289,18 @@ in
             }
           }
         '';
-        storage = "${config.services.home-assistant.configDir}/.storage";
+        storage = "${config.services.home-assistant.configDir}";
-        file = "${storage}/onboarding";
+        file = "${storage}/.storage/onboarding";
       in
         ''
           if ! -f ${file}; then
             mkdir -p ${storage} && cp ${onboarding} ${file}
           fi
-        '');
+        '' + shblib.replaceSecrets {
-
+          userConfig = cfg.config;
-    sops.secrets."home-assistant" = {
+          resultPath = "${config.services.home-assistant.configDir}/secrets.yaml";
-      inherit (cfg) sopsFile;
+          generator = lib.generators.toYAML {};
-      mode = "0440";
+        });
-      owner = "hass";
-      group = "hass";
-      path = "${config.services.home-assistant.configDir}/secrets.yaml";
-      restartUnits = [ "home-assistant.service" ];
-    };
 
     systemd.tmpfiles.rules = [
       "f ${config.services.home-assistant.configDir}/automations.yaml 0755 hass hass"

modules/services/jellyfin.nix

@@ -348,19 +348,33 @@ in
           </BrandingOptions>
         '';
       in
-        shblib.template ldapConfig "/var/lib/jellyfin/plugins/configurations/LDAP-Auth.xml" {
+        shblib.replaceSecretsScript {
+          file = ldapConfig;
+          resultPath = "/var/lib/jellyfin/plugins/configurations/LDAP-Auth.xml";
+          userConfig = {
             "%LDAP_PASSWORD%" = "$(cat ${cfg.ldapPasswordFile})";
+          };
         }
-        + shblib.template ssoConfig "/var/lib/jellyfin/plugins/configurations/SSO-Auth.xml" {
+        + shblib.replaceSecretsScript {
+          file = ssoConfig;
+          resultPath = "/var/lib/jellyfin/plugins/configurations/SSO-Auth.xml";
+          userConfig = {
             "%SSO_SECRET%" = "$(cat ${cfg.ssoSecretFile})";
+          };
         }
-        + shblib.template brandingConfig "/var/lib/jellyfin/config/branding.xml" {"%a%" = "%a%";};
+        + shblib.replaceSecretsScript {
+          file = brandingConfig;
+          resultPath = "/var/lib/jellyfin/config/branding.xml";
+          userConfig = {
+            "%a%" = "%a%";
+          };
+        };
 
     shb.authelia.oidcClients = [
       {
         id = cfg.oidcClientID;
         description = "Jellyfin";
-        secretFile = cfg.ssoSecretFile;
+        secret.source = cfg.ssoSecretFile;
         public = false;
         authorization_policy = "one_factor";
         redirect_uris = [ "https://${cfg.subdomain}.${cfg.domain}/sso/OID/r/${cfg.oidcProvider}" ];

modules/services/nextcloud-server.nix

@@ -829,7 +829,7 @@ in
         {
           id = cfg.apps.sso.clientID;
           description = "Nextcloud";
-          secretFile = cfg.apps.sso.secretFileForAuthelia;
+          secret.source = cfg.apps.sso.secretFileForAuthelia;
           public = "false";
           authorization_policy = cfg.apps.sso.authorization_policy;
           redirect_uris = [ "${protocol}://${fqdnWithPort}/apps/oidc_login/oidc" ];

modules/services/vaultwarden.nix

@@ -148,15 +148,14 @@ in
       "f /var/lib/bitwarden_rs/vaultwarden.env 0640 vaultwarden vaultwarden"
     ];
     systemd.services.vaultwarden.preStart =
-      let
+      shblib.replaceSecrets {
-        envFile = pkgs.writeText "vaultwarden.env" ''
+        userConfig = {
-        DATABASE_URL=postgresql://vaultwarden:%DB_PASSWORD%@127.0.0.1:5432/vaultwarden
+          DATABASE_URL.source = cfg.databasePasswordFile;
-        SMTP_PASSWORD=%SMTP_PASSWORD%
+          DATABASE_URL.transform = v: "postgresql://vaultwarden:${v}@127.0.0.1:5432/vaultwarden";
-        '';
+          SMTP_PASSWORD.source = cfg.smtp.passwordFile;
-      in
+        };
-        shblib.template envFile "/var/lib/bitwarden_rs/vaultwarden.env" {
+        resultPath = "/var/lib/bitwarden_rs/vaultwarden.env";
-          "%DB_PASSWORD%" = "$(cat ${cfg.databasePasswordFile})";
+        generator = v: lib.generators.toINIWithGlobalSection {} { globalSection = v; };
-          "%SMTP_PASSWORD%" = "$(cat ${cfg.smtp.passwordFile})";
       };
 
     shb.nginx.autheliaProtect = [

test/modules/lib.nix

@@ -0,0 +1,110 @@
+{ pkgs, lib, ... }:
+let
+  shblib = pkgs.callPackage ../../lib {};
+in
+{
+  # Tests that withReplacements can:
+  # - recurse in attrs and lists
+  # - .source field is understood
+  # - .transform field is understood
+  # - if .source field is found, ignores other fields
+  testLibWithReplacements = {
+    expected =
+      let
+        item = root: {
+          a = "A";
+          b = "%SECRET_${root}B%";
+          c = "%SECRET_${root}C%";
+        };
+      in
+        (item "") // {
+          nestedAttr = item "NESTEDATTR_";
+          nestedList = [ (item "NESTEDLIST_0_") ];
+          doubleNestedList = [ { n = (item "DOUBLENESTEDLIST_0_N_"); } ];
+        };
+    expr =
+      let
+        item = {
+          a = "A";
+          b.source = "/path/B";
+          b.transform = null;
+          c.source = "/path/C";
+          c.transform = v: "prefix-${v}-suffix";
+          c.other = "other";
+        };
+      in
+        shblib.withReplacements (
+          item // {
+            nestedAttr = item;
+            nestedList = [ item ];
+            doubleNestedList = [ { n = item; } ];
+          }
+        );
+  };
+
+  testLibWithReplacementsRootList = {
+    expected =
+      let
+        item = root: {
+          a = "A";
+          b = "%SECRET_${root}B%";
+          c = "%SECRET_${root}C%";
+        };
+      in
+        [
+          (item "0_")
+          (item "1_")
+          [ (item "2_0_") ]
+          [ { n = (item "3_0_N_"); } ]
+        ];
+    expr =
+      let
+        item = {
+          a = "A";
+          b.source = "/path/B";
+          b.transform = null;
+          c.source = "/path/C";
+          c.transform = v: "prefix-${v}-suffix";
+          c.other = "other";
+        };
+      in
+        shblib.withReplacements [
+          item
+          item
+          [ item ]
+          [ { n = item; } ]
+        ];
+  };
+
+  testLibGetReplacements = {
+    expected =
+      let
+        secrets = root: {
+          "%SECRET_${root}B%" = "$(cat /path/B)";
+          "%SECRET_${root}C%" = "prefix-$(cat /path/C)-suffix";
+        };
+      in
+        (secrets "") //
+        (secrets "NESTEDATTR_") //
+        (secrets "NESTEDLIST_0_") //
+        (secrets "DOUBLENESTEDLIST_0_N_");
+    expr =
+      let
+        item = {
+          a = "A";
+          b.source = "/path/B";
+          b.transform = null;
+          c.source = "/path/C";
+          c.transform = v: "prefix-${v}-suffix";
+          c.other = "other";
+        };
+      in
+        shblib.getReplacements (
+          item // {
+            nestedAttr = item;
+            nestedList = [ item ];
+            doubleNestedList = [ { n = item; } ];
+          }
+        );
+  };
+}

test/vm/authelia.nix

@@ -10,7 +10,6 @@ in
       imports = [
         {
           options = {
-            shb.ssl.enable = lib.mkEnableOption "ssl";
             shb.backup = lib.mkOption { type = lib.types.anything; };
           };
         }
@@ -49,7 +48,7 @@ in
           {
             id = "client1";
             description = "My Client 1";
-            secretFile = pkgs.writeText "secret" "mysecuresecret";
+            secret.source = pkgs.writeText "secret" "mysecuresecret";
             public = false;
             authorization_policy = "one_factor";
             redirect_uris = [ "http://client1.machine/redirect" ];
@@ -57,7 +56,7 @@ in
           {
             id = "client2";
             description = "My Client 2";
-            secretFile = pkgs.writeText "secret" "myothersecret";
+            secret.source = pkgs.writeText "secret" "myothersecret";
             public = false;
             authorization_policy = "one_factor";
             redirect_uris = [ "http://client2.machine/redirect" ];

test/vm/lib.nix

@@ -0,0 +1,82 @@
+{ pkgs, lib, ... }:
+let
+  shblib = pkgs.callPackage ../../lib {};
+in
+{
+  template =
+    let
+      aSecret = pkgs.writeText "a-secret.txt" "Secret of A";
+      bSecret = pkgs.writeText "b-secret.txt" "Secret of B";
+      userConfig = {
+        a.a.source = aSecret;
+        b.source = bSecret;
+        b.transform = v: "prefix-${v}-suffix";
+        c = "not secret C";
+        d.d = "not secret D";
+      };
+
+      wantedConfig = {
+        a.a = "Secret of A";
+        b = "prefix-Secret of B-suffix";
+        c = "not secret C";
+        d.d = "not secret D";
+      };
+
+      configWithTemplates = shblib.withReplacements userConfig;
+
+      nonSecretConfigFile = pkgs.writeText "config.yaml.template" (lib.generators.toJSON {} configWithTemplates);
+
+      replacements = shblib.getReplacements userConfig;
+
+      replaceInTemplate = shblib.replaceSecretsScript {
+        file = nonSecretConfigFile;
+        resultPath = "/var/lib/config.yaml";
+        inherit replacements;
+      };
+
+      replaceInTemplate2 = shblib.replaceSecrets {
+        inherit userConfig;
+        resultPath = "/var/lib/config2.yaml";
+        generator = lib.generators.toJSON {};
+      };
+    in
+      pkgs.nixosTest {
+        name = "lib-template";
+        nodes.machine = { config, pkgs, ... }:
+          {
+            imports = [
+              {
+                options = {
+                  libtest.config = lib.mkOption {
+                    type = lib.types.attrsOf (lib.types.oneOf [ lib.types.str shblib.secretFileType ]);
+                  };
+                };
+              }
+            ];
+
+            system.activationScripts = {
+              libtest = replaceInTemplate;
+              libtest2 = replaceInTemplate2;
+            };
+          };
+
+        testScript = { nodes, ... }: ''
+        import json
+        start_all()
+
+        wantedConfig = json.loads('${lib.generators.toJSON {} wantedConfig}')
+        gotConfig = json.loads(machine.succeed("cat /var/lib/config.yaml"))
+        gotConfig2 = json.loads(machine.succeed("cat /var/lib/config2.yaml"))
+
+        # For debugging purpose
+        print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
+        print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate2" replaceInTemplate2}"))
+
+        if wantedConfig != gotConfig:
+          raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+
+        if wantedConfig != gotConfig2:
+          raise Exception("\nwantedConfig:  {}\n!= gotConfig2: {}".format(wantedConfig, gotConfig))
+        '';
+      };
+}