diff --git a/lib/default.nix b/lib/default.nix index 0d26ebf..19831ca 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -1,10 +1,15 @@ { pkgs, lib }: rec { + # Replace secrets in a file. + # - userConfig is an attrset that will produce a config file. + # - resultPath is the location the config file should have on the filesystem. + # - generator is a function taking two arguments name and value and returning path in the nix + # nix store where the replaceSecrets = { userConfig, resultPath, generator }: let configWithTemplates = withReplacements userConfig; - nonSecretConfigFile = pkgs.writeText "${resultPath}.template" (generator "template" configWithTemplates); + nonSecretConfigFile = generator "template" configWithTemplates; replacements = getReplacements userConfig; in @@ -13,6 +18,9 @@ rec { inherit resultPath replacements; }; + replaceSecretsFormatAdapter = format: format.generate; + replaceSecretsGeneratorAdapter = generator: name: value: pkgs.writeText "generator " (generator value); + template = file: newPath: replacements: replaceSecretsScript { inherit file replacements; resultPath = newPath; @@ -22,6 +30,9 @@ rec { let templatePath = resultPath + ".template"; sedPatterns = lib.strings.concatStringsSep " " (lib.attrsets.mapAttrsToList (from: to: "-e \"s|${from}|${to}|\"") replacements); + sedCmd = if replacements == {} + then "cat" + else "${pkgs.gnused}/bin/sed ${sedPatterns}"; in '' set -euo pipefail @@ -29,11 +40,7 @@ rec { mkdir -p $(dirname ${templatePath}) ln -fs ${file} ${templatePath} rm -f ${resultPath} - if [ -z "${sedPatterns}" ]; then - cat ${templatePath} > ${resultPath} - else - ${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath} > ${resultPath} - fi + ${sedCmd} ${templatePath} > ${resultPath} ''; secretFileType = lib.types.submodule { @@ -115,4 +122,86 @@ rec { lib.lists.concatMap (collect pred) attrs else []; + + # Generator for XML + formatXML = { + enclosingRoot ? null + }: { + type = with lib.types; let + valueType = nullOr (oneOf [ + bool + int + float + str + path + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "XML value"; + }; + in valueType; + + generate = name: value: pkgs.callPackage ({ runCommand, python3 }: runCommand "config" { + value = builtins.toJSON ( + if enclosingRoot == null then + value + else + { ${enclosingRoot} = value; }); + passAsFile = [ "value" ]; + } (pkgs.writers.writePython3 "dict2xml" { + libraries = with python3.pkgs; [ python dict2xml ]; + } '' + import os + import json + from dict2xml import dict2xml + + with open(os.environ["valuePath"]) as f: + content = json.loads(f.read()) + if content is None: + print("Could not parse env var valuePath as json") + os.exit(2) + with open(os.environ["out"], "w") as out: + out.write(dict2xml(content)) + '')) {}; + + }; + + parseXML = xml: + let + xmlToJsonFile = pkgs.callPackage ({ runCommand, python3 }: runCommand "config" { + inherit xml; + passAsFile = [ "xml" ]; + } (pkgs.writers.writePython3 "xml2json" { + libraries = with python3.pkgs; [ python ]; + } '' + import os + import json + from collections import ChainMap + from xml.etree import ElementTree + + + def xml_to_dict_recursive(root): + all_descendants = list(root) + if len(all_descendants) == 0: + return {root.tag: root.text} + else: + merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants)) + return {root.tag: dict(merged_dict)} + + + with open(os.environ["xmlPath"]) as f: + root = ElementTree.XML(f.read()) + xml = xml_to_dict_recursive(root) + j = json.dumps(xml) + + with open(os.environ["out"], "w") as out: + out.write(j) + '')) {}; + in + builtins.fromJSON (builtins.readFile xmlToJsonFile); + + renameAttrName = attrset: from: to: + (lib.attrsets.filterAttrs (name: v: name == from) attrset) // { + ${to} = attrset.${from}; + }; } diff --git a/modules/blocks/authelia.nix b/modules/blocks/authelia.nix index ac7d26a..3670c20 100644 --- a/modules/blocks/authelia.nix +++ b/modules/blocks/authelia.nix @@ -341,7 +341,7 @@ in identity_providers.oidc.clients = clients; }; resultPath = "/var/lib/authelia-${fqdn}/oidc_clients.yaml"; - generator = name: value: lib.generators.toYAML {} value; + generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toYAML {}); }; in lib.mkBefore (mkCfg cfg.oidcClients); diff --git a/modules/services/arr.nix b/modules/services/arr.nix index 7b4200b..8b0db9d 100644 --- a/modules/services/arr.nix +++ b/modules/services/arr.nix @@ -8,7 +8,7 @@ let apps = { radarr = { - settingsFormat = formatXML {}; + settingsFormat = shblib.formatXML { enclosingRoot = "Config"; }; moreOptions = { settings = lib.mkOption { description = "Specific options for radarr."; @@ -16,7 +16,7 @@ let type = lib.types.submodule { freeformType = apps.radarr.settingsFormat.type; options = { - APIKey = lib.mkOption { + ApiKey = lib.mkOption { type = shblib.secretFileType; description = "Path to api key secret file."; }; @@ -66,7 +66,7 @@ let }; }; sonarr = { - settingsFormat = formatXML {}; + settingsFormat = shblib.formatXML { enclosingRoot = "Config"; }; moreOptions = { settings = lib.mkOption { description = "Specific options for sonarr."; @@ -74,7 +74,7 @@ let type = lib.types.submodule { freeformType = apps.sonarr.settingsFormat.type; options = { - APIKey = lib.mkOption { + ApiKey = lib.mkOption { type = shblib.secretFileType; description = "Path to api key secret file."; }; @@ -119,7 +119,7 @@ let }; }; bazarr = { - settingsFormat = formatXML {}; + settingsFormat = shblib.formatXML { enclosingRoot = "Config"; }; moreOptions = { settings = lib.mkOption { description = "Specific options for bazarr."; @@ -144,7 +144,7 @@ let }; }; readarr = { - settingsFormat = formatXML {}; + settingsFormat = shblib.formatXML { enclosingRoot = "Config"; }; moreOptions = { settings = lib.mkOption { description = "Specific options for readarr."; @@ -168,7 +168,7 @@ let }; }; lidarr = { - settingsFormat = formatXML {}; + settingsFormat = shblib.formatXML { enclosingRoot = "Config"; }; moreOptions = { settings = lib.mkOption { description = "Specific options for lidarr."; @@ -200,7 +200,7 @@ let type = lib.types.submodule { freeformType = apps.jackett.settingsFormat.type; options = { - APIKey = lib.mkOption { + ApiKey = lib.mkOption { type = shblib.secretFileType; description = "Path to api key secret file."; }; @@ -291,42 +291,6 @@ let }; }; - formatXML = {}: { - type = with lib.types; let - valueType = nullOr (oneOf [ - bool - int - float - str - path - (attrsOf valueType) - (listOf valueType) - ]) // { - description = "XML value"; - }; - in valueType; - - generate = name: value: builtins.readFile (pkgs.callPackage ({ runCommand, python3 }: runCommand "config" { - value = builtins.toJSON {Config = value;}; - passAsFile = [ "value" ]; - } (pkgs.writers.writePython3 "dict2xml" { - libraries = with python3.pkgs; [ python dict2xml ]; - } '' - import os - import json - from dict2xml import dict2xml - - with open(os.environ["valuePath"]) as f: - content = json.loads(f.read()) - if content is None: - print("Could not parse env var valuePath as json") - os.exit(2) - with open(os.environ["out"], "w") as out: - out.write(dict2xml(content)) - '')) {}); - - }; - appOption = name: c: lib.nameValuePair name (lib.mkOption { description = "Configuration for ${name}"; default = {}; @@ -402,7 +366,7 @@ in AuthenticationMethod = "External"; }); resultPath = "${config.services.radarr.dataDir}/config.xml"; - generator = apps.radarr.settingsFormat.generate; + generator = shblib.replaceSecretsFormatAdapter apps.radarr.settingsFormat; }; shb.nginx.autheliaProtect = [ (autheliaProtect {} cfg') ]; @@ -563,7 +527,7 @@ in extraGroups = [ "media" ]; }; systemd.services.jackett.preStart = shblib.replaceSecrets { - userConfig = cfg'.settings; + userConfig = shblib.renameAttrName cfg'.settings "ApiKey" "APIKey"; resultPath = "${config.services.jackett.dataDir}/ServerConfig.json"; generator = apps.jackett.settingsFormat.generate; }; diff --git a/test/modules/arr.nix b/test/modules/arr.nix index 2fbe58f..2ca7d39 100644 --- a/test/modules/arr.nix +++ b/test/modules/arr.nix @@ -126,7 +126,7 @@ in enable = true; authEndpoint = "https://oidc.example.com"; settings = { - APIKey.source = pkgs.writeText "key" "/run/radarr/apikey"; + ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey"; }; }; }; @@ -199,7 +199,7 @@ in enable = true; authEndpoint = "https://oidc.example.com"; settings = { - APIKey.source = pkgs.writeText "key" "/run/radarr/apikey"; + ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey"; }; backupCfg = { enable = true; diff --git a/test/modules/lib.nix b/test/modules/lib.nix index a34dcf5..0a8227e 100644 --- a/test/modules/lib.nix +++ b/test/modules/lib.nix @@ -107,4 +107,22 @@ in } ); }; + + testParseXML = { + expected = { + "a" = { + "b" = "1"; + "c" = { + "d" = "1"; + }; + }; + }; + + expr = shblib.parseXML '' + + 1 + 1 + + ''; + }; } diff --git a/test/vm/arr.nix b/test/vm/arr.nix index a14b75c..e4e8be4 100644 --- a/test/vm/arr.nix +++ b/test/vm/arr.nix @@ -2,9 +2,11 @@ let pkgs' = pkgs; # TODO: Test login - commonTestScript = appname: { nodes, ... }: + commonTestScript = appname: cfgPathFn: { nodes, ... }: let shbapp = nodes.server.shb.arr.${appname}; + cfgPath = cfgPathFn shbapp; + apiKey = if (shbapp.settings ? ApiKey) then "01234567890123456789" else null; hasSSL = !(isNull shbapp.ssl); fqdn = if hasSSL then "https://${appname}.example.com" else "http://${appname}.example.com"; healthUrl = "/health"; @@ -48,9 +50,15 @@ let if response['code'] != 200: raise Exception(f"Code is {response['code']}") + '' + lib.optionalString (apiKey != null) '' + + with subtest("apikey"): + config = server.succeed("cat ${cfgPath}") + if "${apiKey}" not in config: + raise Exception(f"Unexpected API Key. Want '${apiKey}', got '{config}'") ''; - basic = appname: pkgs.testers.runNixOSTest { + basic = appname: cfgPathFn: pkgs.testers.runNixOSTest { name = "arr-${appname}-basic"; nodes.server = { config, pkgs, ... }: { @@ -73,7 +81,7 @@ let domain = "example.com"; subdomain = appname; - settings.APIKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters. + settings.ApiKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters. }; # Nginx port. networking.firewall.allowedTCPPorts = [ 80 ]; @@ -81,14 +89,14 @@ let nodes.client = {}; - testScript = commonTestScript appname; + testScript = commonTestScript appname cfgPathFn; }; in { - radarr_basic = basic "radarr"; - sonarr_basic = basic "sonarr"; - bazarr_basic = basic "bazarr"; - readarr_basic = basic "readarr"; - lidarr_basic = basic "lidarr"; - jackett_basic = basic "jackett"; + radarr_basic = basic "radarr" (cfg: "${cfg.dataDir}/config.xml"); + sonarr_basic = basic "sonarr" (cfg: "${cfg.dataDir}/config.xml"); + bazarr_basic = basic "bazarr" (cfg: "/var/lib/bazarr/config.xml"); + readarr_basic = basic "readarr" (cfg: "${cfg.dataDir}/config.xml"); + lidarr_basic = basic "lidarr" (cfg: "${cfg.dataDir}/config.xml"); + jackett_basic = basic "jackett" (cfg: "${cfg.dataDir}/ServerConfig.json"); } diff --git a/test/vm/lib.nix b/test/vm/lib.nix index 681e89e..d594650 100644 --- a/test/vm/lib.nix +++ b/test/vm/lib.nix @@ -36,10 +36,22 @@ in inherit replacements; }; - replaceInTemplate2 = shblib.replaceSecrets { + replaceInTemplateJSON = shblib.replaceSecrets { inherit userConfig; - resultPath = "/var/lib/config2.yaml"; - generator = name: value: lib.generators.toJSON {} value; + resultPath = "/var/lib/config.json"; + generator = shblib.replaceSecretsFormatAdapter (pkgs.formats.json {}); + }; + + replaceInTemplateJSONGen = shblib.replaceSecrets { + inherit userConfig; + resultPath = "/var/lib/config_gen.json"; + generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toJSON {}); + }; + + replaceInTemplateXML = shblib.replaceSecrets { + inherit userConfig; + resultPath = "/var/lib/config.xml"; + generator = shblib.replaceSecretsFormatAdapter (shblib.formatXML {enclosingRoot = "Root";}); }; in pkgs.testers.runNixOSTest { @@ -60,27 +72,72 @@ in system.activationScripts = { libtest = replaceInTemplate; - libtest2 = replaceInTemplate2; + libtestJSON = replaceInTemplateJSON; + libtestJSONGen = replaceInTemplateJSONGen; + libtestXML = replaceInTemplateXML; }; }; testScript = { nodes, ... }: '' import json + from collections import ChainMap + from xml.etree import ElementTree + start_all() + machine.wait_for_file("/var/lib/config.yaml") + machine.wait_for_file("/var/lib/config.json") + machine.wait_for_file("/var/lib/config_gen.json") + machine.wait_for_file("/var/lib/config.xml") + + def xml_to_dict_recursive(root): + all_descendants = list(root) + if len(all_descendants) == 0: + return {root.tag: root.text} + else: + merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants)) + return {root.tag: dict(merged_dict)} wantedConfig = json.loads('${lib.generators.toJSON {} wantedConfig}') - gotConfig = json.loads(machine.succeed("cat /var/lib/config.yaml")) - gotConfig2 = json.loads(machine.succeed("cat /var/lib/config2.yaml")) - # For debugging purpose - print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}")) - print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate2" replaceInTemplate2}")) + with subtest("config"): + print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}")) - if wantedConfig != gotConfig: - raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig)) + gotConfig = machine.succeed("cat /var/lib/config.yaml") + print(gotConfig) + gotConfig = json.loads(gotConfig) - if wantedConfig != gotConfig2: - raise Exception("\nwantedConfig: {}\n!= gotConfig2: {}".format(wantedConfig, gotConfig)) + if wantedConfig != gotConfig: + raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig)) + + with subtest("config JSON Gen"): + print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSONGen" replaceInTemplateJSONGen}")) + + gotConfig = machine.succeed("cat /var/lib/config_gen.json") + print(gotConfig) + gotConfig = json.loads(gotConfig) + + if wantedConfig != gotConfig: + raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig)) + + with subtest("config JSON"): + print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSON" replaceInTemplateJSON}")) + + gotConfig = machine.succeed("cat /var/lib/config.json") + print(gotConfig) + gotConfig = json.loads(gotConfig) + + if wantedConfig != gotConfig: + raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig)) + + with subtest("config XML"): + print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateXML" replaceInTemplateXML}")) + + gotConfig = machine.succeed("cat /var/lib/config.xml") + print(gotConfig) + gotConfig = xml_to_dict_recursive(ElementTree.XML(gotConfig))['Root'] + + if wantedConfig != gotConfig: + raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig)) ''; }; }