make config with secrets correctly generated

lib/default.nix

@@ -1,10 +1,15 @@
 { pkgs, lib }:
 rec {
+  # Replace secrets in a file.
+  # - userConfig is an attrset that will produce a config file.
+  # - resultPath is the location the config file should have on the filesystem.
+  # - generator is a function taking two arguments name and value and returning path in the nix
+  #   nix store where the
   replaceSecrets = { userConfig, resultPath, generator }:
     let
       configWithTemplates = withReplacements userConfig;
 
-      nonSecretConfigFile = pkgs.writeText "${resultPath}.template" (generator "template" configWithTemplates);
+      nonSecretConfigFile = generator "template" configWithTemplates;
 
       replacements = getReplacements userConfig;
     in
@@ -13,6 +18,9 @@ rec {
         inherit resultPath replacements;
       };
 
+  replaceSecretsFormatAdapter = format: format.generate;
+  replaceSecretsGeneratorAdapter = generator: name: value: pkgs.writeText "generator " (generator value);
+
   template = file: newPath: replacements: replaceSecretsScript {
     inherit file replacements;
     resultPath = newPath;
@@ -22,6 +30,9 @@ rec {
     let
       templatePath = resultPath + ".template";
       sedPatterns = lib.strings.concatStringsSep " " (lib.attrsets.mapAttrsToList (from: to: "-e \"s|${from}|${to}|\"") replacements);
+      sedCmd = if replacements == {}
+               then "cat"
+               else "${pkgs.gnused}/bin/sed ${sedPatterns}";
     in
       ''
       set -euo pipefail
@@ -29,11 +40,7 @@ rec {
       mkdir -p $(dirname ${templatePath})
       ln -fs ${file} ${templatePath}
       rm -f ${resultPath}
-      if [ -z "${sedPatterns}" ]; then
+      ${sedCmd} ${templatePath} > ${resultPath}
-        cat ${templatePath} > ${resultPath}
-      else
-        ${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath} > ${resultPath}
-      fi
       '';
 
   secretFileType = lib.types.submodule {
@@ -115,4 +122,86 @@ rec {
       lib.lists.concatMap (collect pred) attrs
     else
       [];
+
+  # Generator for XML
+  formatXML = {
+    enclosingRoot ? null
+  }: {
+    type = with lib.types; let
+      valueType = nullOr (oneOf [
+        bool
+        int
+        float
+        str
+        path
+        (attrsOf valueType)
+        (listOf valueType)
+      ]) // {
+        description = "XML value";
+      };
+    in valueType;
+
+    generate = name: value: pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
+      value = builtins.toJSON (
+        if enclosingRoot == null then
+          value
+        else
+          { ${enclosingRoot} = value; });
+      passAsFile = [ "value" ];
+    } (pkgs.writers.writePython3 "dict2xml" {
+      libraries = with python3.pkgs; [ python dict2xml ];
+    } ''
+      import os
+      import json
+      from dict2xml import dict2xml
+
+      with open(os.environ["valuePath"]) as f:
+          content = json.loads(f.read())
+          if content is None:
+              print("Could not parse env var valuePath as json")
+              os.exit(2)
+          with open(os.environ["out"], "w") as out:
+              out.write(dict2xml(content))
+    '')) {};
+
+  };
+
+  parseXML = xml:
+    let
+      xmlToJsonFile = pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
+        inherit xml;
+        passAsFile = [ "xml" ];
+      } (pkgs.writers.writePython3 "xml2json" {
+        libraries = with python3.pkgs; [ python ];
+      } ''
+        import os
+        import json
+        from collections import ChainMap
+        from xml.etree import ElementTree
+        
+
+        def xml_to_dict_recursive(root):
+            all_descendants = list(root)
+            if len(all_descendants) == 0:
+                return {root.tag: root.text}
+            else:
+                merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
+                return {root.tag: dict(merged_dict)}
+
+
+        with open(os.environ["xmlPath"]) as f:
+            root = ElementTree.XML(f.read())
+            xml = xml_to_dict_recursive(root)
+            j = json.dumps(xml)
+
+            with open(os.environ["out"], "w") as out:
+                out.write(j)
+      '')) {};
+    in
+      builtins.fromJSON (builtins.readFile xmlToJsonFile);
+
+  renameAttrName = attrset: from: to:
+    (lib.attrsets.filterAttrs (name: v: name == from) attrset) // {
+      ${to} = attrset.${from};
+    };
 }

modules/blocks/authelia.nix

@@ -341,7 +341,7 @@ in
               identity_providers.oidc.clients = clients;
             };
             resultPath = "/var/lib/authelia-${fqdn}/oidc_clients.yaml";
-            generator = name: value: lib.generators.toYAML {} value;
+            generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toYAML {});
           };
       in
         lib.mkBefore (mkCfg cfg.oidcClients);

modules/services/arr.nix

@@ -8,7 +8,7 @@ let
 
   apps = {
     radarr = {
-      settingsFormat = formatXML {};
+      settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
       moreOptions = {
         settings = lib.mkOption {
           description = "Specific options for radarr.";
@@ -16,7 +16,7 @@ let
           type = lib.types.submodule {
             freeformType = apps.radarr.settingsFormat.type;
             options = {
-              APIKey = lib.mkOption {
+              ApiKey = lib.mkOption {
                 type = shblib.secretFileType;
                 description = "Path to api key secret file.";
               };
@@ -66,7 +66,7 @@ let
       };
     };
     sonarr = {
-      settingsFormat = formatXML {};
+      settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
       moreOptions = {
         settings = lib.mkOption {
           description = "Specific options for sonarr.";
@@ -74,7 +74,7 @@ let
           type = lib.types.submodule {
             freeformType = apps.sonarr.settingsFormat.type;
             options = {
-              APIKey = lib.mkOption {
+              ApiKey = lib.mkOption {
                 type = shblib.secretFileType;
                 description = "Path to api key secret file.";
               };
@@ -119,7 +119,7 @@ let
       };
     };
     bazarr = {
-      settingsFormat = formatXML {};
+      settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
       moreOptions = {
         settings = lib.mkOption {
           description = "Specific options for bazarr.";
@@ -144,7 +144,7 @@ let
       };
     };
     readarr = {
-      settingsFormat = formatXML {};
+      settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
       moreOptions = {
         settings = lib.mkOption {
           description = "Specific options for readarr.";
@@ -168,7 +168,7 @@ let
       };
     };
     lidarr = {
-      settingsFormat = formatXML {};
+      settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
       moreOptions = {
         settings = lib.mkOption {
           description = "Specific options for lidarr.";
@@ -200,7 +200,7 @@ let
           type = lib.types.submodule {
             freeformType = apps.jackett.settingsFormat.type;
             options = {
-              APIKey = lib.mkOption {
+              ApiKey = lib.mkOption {
                 type = shblib.secretFileType;
                 description = "Path to api key secret file.";
               };
@@ -291,42 +291,6 @@ let
     };
   };
 
-  formatXML = {}: {
-    type = with lib.types; let
-      valueType = nullOr (oneOf [
-        bool
-        int
-        float
-        str
-        path
-        (attrsOf valueType)
-        (listOf valueType)
-      ]) // {
-        description = "XML value";
-      };
-    in valueType;
-
-    generate = name: value: builtins.readFile (pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
-      value = builtins.toJSON {Config = value;};
-      passAsFile = [ "value" ];
-    } (pkgs.writers.writePython3 "dict2xml" {
-      libraries = with python3.pkgs; [ python dict2xml ];
-    } ''
-      import os
-      import json
-      from dict2xml import dict2xml
-
-      with open(os.environ["valuePath"]) as f:
-          content = json.loads(f.read())
-          if content is None:
-              print("Could not parse env var valuePath as json")
-              os.exit(2)
-          with open(os.environ["out"], "w") as out:
-              out.write(dict2xml(content))
-    '')) {});
-
-  };
-
   appOption = name: c: lib.nameValuePair name (lib.mkOption {
     description = "Configuration for ${name}";
     default = {};
@@ -402,7 +366,7 @@ in
                        AuthenticationMethod = "External";
                      });
         resultPath = "${config.services.radarr.dataDir}/config.xml";
-        generator = apps.radarr.settingsFormat.generate;
+        generator = shblib.replaceSecretsFormatAdapter apps.radarr.settingsFormat;
       };
 
       shb.nginx.autheliaProtect = [ (autheliaProtect {} cfg') ];
@@ -563,7 +527,7 @@ in
         extraGroups = [ "media" ];
       };
       systemd.services.jackett.preStart = shblib.replaceSecrets {
-        userConfig = cfg'.settings;
+        userConfig = shblib.renameAttrName cfg'.settings "ApiKey" "APIKey";
         resultPath = "${config.services.jackett.dataDir}/ServerConfig.json";
         generator = apps.jackett.settingsFormat.generate;
       };

test/modules/arr.nix

@@ -126,7 +126,7 @@ in
         enable = true;
         authEndpoint = "https://oidc.example.com";
         settings = {
-          APIKey.source = pkgs.writeText "key" "/run/radarr/apikey";
+          ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey";
         };
       };
     };
@@ -199,7 +199,7 @@ in
         enable = true;
         authEndpoint = "https://oidc.example.com";
         settings = {
-          APIKey.source = pkgs.writeText "key" "/run/radarr/apikey";
+          ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey";
         };
         backupCfg = {
           enable = true;

test/modules/lib.nix

@@ -107,4 +107,22 @@ in
           }
         );
   };
+
+  testParseXML = {
+    expected = {
+      "a" = {
+        "b" = "1";
+        "c" = {
+          "d" = "1";
+        };
+      };
+    };
+
+    expr = shblib.parseXML ''
+    <a>
+      <b>1</b>
+      <c><d>1</d></c>
+    </a>
+    '';
+  };
 }

test/vm/arr.nix

@@ -2,9 +2,11 @@
 let
   pkgs' = pkgs;
   # TODO: Test login
-  commonTestScript = appname: { nodes, ... }:
+  commonTestScript = appname: cfgPathFn: { nodes, ... }:
     let
       shbapp = nodes.server.shb.arr.${appname};
+      cfgPath = cfgPathFn shbapp;
+      apiKey = if (shbapp.settings ? ApiKey) then "01234567890123456789" else null;
       hasSSL = !(isNull shbapp.ssl);
       fqdn = if hasSSL then "https://${appname}.example.com" else "http://${appname}.example.com";
       healthUrl = "/health";
@@ -48,9 +50,15 @@ let
 
         if response['code'] != 200:
             raise Exception(f"Code is {response['code']}")
+    '' + lib.optionalString (apiKey != null) ''
+
+    with subtest("apikey"):
+        config = server.succeed("cat ${cfgPath}")
+        if "${apiKey}" not in config:
+            raise Exception(f"Unexpected API Key. Want '${apiKey}', got '{config}'")
     '';
 
-  basic = appname: pkgs.testers.runNixOSTest {
+  basic = appname: cfgPathFn: pkgs.testers.runNixOSTest {
     name = "arr-${appname}-basic";
 
     nodes.server = { config, pkgs, ... }: {
@@ -73,7 +81,7 @@ let
         domain = "example.com";
         subdomain = appname;
 
-        settings.APIKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters.
+        settings.ApiKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters.
       };
       # Nginx port.
       networking.firewall.allowedTCPPorts = [ 80 ];
@@ -81,14 +89,14 @@ let
 
     nodes.client = {};
 
-    testScript = commonTestScript appname;
+    testScript = commonTestScript appname cfgPathFn;
   };
 in
 {
-  radarr_basic = basic "radarr";
+  radarr_basic = basic "radarr" (cfg: "${cfg.dataDir}/config.xml");
-  sonarr_basic = basic "sonarr";
+  sonarr_basic = basic "sonarr" (cfg: "${cfg.dataDir}/config.xml");
-  bazarr_basic = basic "bazarr";
+  bazarr_basic = basic "bazarr" (cfg: "/var/lib/bazarr/config.xml");
-  readarr_basic = basic "readarr";
+  readarr_basic = basic "readarr" (cfg: "${cfg.dataDir}/config.xml");
-  lidarr_basic = basic "lidarr";
+  lidarr_basic = basic "lidarr" (cfg: "${cfg.dataDir}/config.xml");
-  jackett_basic = basic "jackett";
+  jackett_basic = basic "jackett" (cfg: "${cfg.dataDir}/ServerConfig.json");
 }

test/vm/lib.nix

@@ -36,10 +36,22 @@ in
         inherit replacements;
       };
 
-      replaceInTemplate2 = shblib.replaceSecrets {
+      replaceInTemplateJSON = shblib.replaceSecrets {
         inherit userConfig;
-        resultPath = "/var/lib/config2.yaml";
+        resultPath = "/var/lib/config.json";
-        generator = name: value: lib.generators.toJSON {} value;
+        generator = shblib.replaceSecretsFormatAdapter (pkgs.formats.json {});
+      };
+
+      replaceInTemplateJSONGen = shblib.replaceSecrets {
+        inherit userConfig;
+        resultPath = "/var/lib/config_gen.json";
+        generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toJSON {});
+      };
+
+      replaceInTemplateXML = shblib.replaceSecrets {
+        inherit userConfig;
+        resultPath = "/var/lib/config.xml";
+        generator = shblib.replaceSecretsFormatAdapter (shblib.formatXML {enclosingRoot = "Root";});
       };
     in
       pkgs.testers.runNixOSTest {
@@ -60,27 +72,72 @@ in
 
             system.activationScripts = {
               libtest = replaceInTemplate;
-              libtest2 = replaceInTemplate2;
+              libtestJSON = replaceInTemplateJSON;
+              libtestJSONGen = replaceInTemplateJSONGen;
+              libtestXML = replaceInTemplateXML;
             };
           };
 
         testScript = { nodes, ... }: ''
         import json
+        from collections import ChainMap
+        from xml.etree import ElementTree
+
         start_all()
+        machine.wait_for_file("/var/lib/config.yaml")
+        machine.wait_for_file("/var/lib/config.json")
+        machine.wait_for_file("/var/lib/config_gen.json")
+        machine.wait_for_file("/var/lib/config.xml")
+
+        def xml_to_dict_recursive(root):
+            all_descendants = list(root)
+            if len(all_descendants) == 0:
+                return {root.tag: root.text}
+            else:
+                merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
+                return {root.tag: dict(merged_dict)}
 
         wantedConfig = json.loads('${lib.generators.toJSON {} wantedConfig}')
-        gotConfig = json.loads(machine.succeed("cat /var/lib/config.yaml"))
-        gotConfig2 = json.loads(machine.succeed("cat /var/lib/config2.yaml"))
 
-        # For debugging purpose
+        with subtest("config"):
-        print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
+          print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
-        print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate2" replaceInTemplate2}"))
 
-        if wantedConfig != gotConfig:
+          gotConfig = machine.succeed("cat /var/lib/config.yaml")
-          raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+          print(gotConfig)
+          gotConfig = json.loads(gotConfig)
 
-        if wantedConfig != gotConfig2:
+          if wantedConfig != gotConfig:
-          raise Exception("\nwantedConfig:  {}\n!= gotConfig2: {}".format(wantedConfig, gotConfig))
+            raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+
+        with subtest("config JSON Gen"):
+          print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSONGen" replaceInTemplateJSONGen}"))
+
+          gotConfig = machine.succeed("cat /var/lib/config_gen.json")
+          print(gotConfig)
+          gotConfig = json.loads(gotConfig)
+
+          if wantedConfig != gotConfig:
+            raise Exception("\nwantedConfig:  {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+
+        with subtest("config JSON"):
+          print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSON" replaceInTemplateJSON}"))
+
+          gotConfig = machine.succeed("cat /var/lib/config.json")
+          print(gotConfig)
+          gotConfig = json.loads(gotConfig)
+
+          if wantedConfig != gotConfig:
+            raise Exception("\nwantedConfig:  {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+
+        with subtest("config XML"):
+          print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateXML" replaceInTemplateXML}"))
+
+          gotConfig = machine.succeed("cat /var/lib/config.xml")
+          print(gotConfig)
+          gotConfig = xml_to_dict_recursive(ElementTree.XML(gotConfig))['Root']
+
+          if wantedConfig != gotConfig:
+            raise Exception("\nwantedConfig:  {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
         '';
       };
 }