diff --git a/lib/default.nix b/lib/default.nix
index 0d26ebf..19831ca 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,10 +1,15 @@
{ pkgs, lib }:
rec {
+ # Replace secrets in a file.
+ # - userConfig is an attrset that will produce a config file.
+ # - resultPath is the location the config file should have on the filesystem.
+ # - generator is a function taking two arguments name and value and returning path in the nix
+ # nix store where the
replaceSecrets = { userConfig, resultPath, generator }:
let
configWithTemplates = withReplacements userConfig;
- nonSecretConfigFile = pkgs.writeText "${resultPath}.template" (generator "template" configWithTemplates);
+ nonSecretConfigFile = generator "template" configWithTemplates;
replacements = getReplacements userConfig;
in
@@ -13,6 +18,9 @@ rec {
inherit resultPath replacements;
};
+ replaceSecretsFormatAdapter = format: format.generate;
+ replaceSecretsGeneratorAdapter = generator: name: value: pkgs.writeText "generator " (generator value);
+
template = file: newPath: replacements: replaceSecretsScript {
inherit file replacements;
resultPath = newPath;
@@ -22,6 +30,9 @@ rec {
let
templatePath = resultPath + ".template";
sedPatterns = lib.strings.concatStringsSep " " (lib.attrsets.mapAttrsToList (from: to: "-e \"s|${from}|${to}|\"") replacements);
+ sedCmd = if replacements == {}
+ then "cat"
+ else "${pkgs.gnused}/bin/sed ${sedPatterns}";
in
''
set -euo pipefail
@@ -29,11 +40,7 @@ rec {
mkdir -p $(dirname ${templatePath})
ln -fs ${file} ${templatePath}
rm -f ${resultPath}
- if [ -z "${sedPatterns}" ]; then
- cat ${templatePath} > ${resultPath}
- else
- ${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath} > ${resultPath}
- fi
+ ${sedCmd} ${templatePath} > ${resultPath}
'';
secretFileType = lib.types.submodule {
@@ -115,4 +122,86 @@ rec {
lib.lists.concatMap (collect pred) attrs
else
[];
+
+ # Generator for XML
+ formatXML = {
+ enclosingRoot ? null
+ }: {
+ type = with lib.types; let
+ valueType = nullOr (oneOf [
+ bool
+ int
+ float
+ str
+ path
+ (attrsOf valueType)
+ (listOf valueType)
+ ]) // {
+ description = "XML value";
+ };
+ in valueType;
+
+ generate = name: value: pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
+ value = builtins.toJSON (
+ if enclosingRoot == null then
+ value
+ else
+ { ${enclosingRoot} = value; });
+ passAsFile = [ "value" ];
+ } (pkgs.writers.writePython3 "dict2xml" {
+ libraries = with python3.pkgs; [ python dict2xml ];
+ } ''
+ import os
+ import json
+ from dict2xml import dict2xml
+
+ with open(os.environ["valuePath"]) as f:
+ content = json.loads(f.read())
+ if content is None:
+ print("Could not parse env var valuePath as json")
+ os.exit(2)
+ with open(os.environ["out"], "w") as out:
+ out.write(dict2xml(content))
+ '')) {};
+
+ };
+
+ parseXML = xml:
+ let
+ xmlToJsonFile = pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
+ inherit xml;
+ passAsFile = [ "xml" ];
+ } (pkgs.writers.writePython3 "xml2json" {
+ libraries = with python3.pkgs; [ python ];
+ } ''
+ import os
+ import json
+ from collections import ChainMap
+ from xml.etree import ElementTree
+
+
+ def xml_to_dict_recursive(root):
+ all_descendants = list(root)
+ if len(all_descendants) == 0:
+ return {root.tag: root.text}
+ else:
+ merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
+ return {root.tag: dict(merged_dict)}
+
+
+ with open(os.environ["xmlPath"]) as f:
+ root = ElementTree.XML(f.read())
+ xml = xml_to_dict_recursive(root)
+ j = json.dumps(xml)
+
+ with open(os.environ["out"], "w") as out:
+ out.write(j)
+ '')) {};
+ in
+ builtins.fromJSON (builtins.readFile xmlToJsonFile);
+
+ renameAttrName = attrset: from: to:
+ (lib.attrsets.filterAttrs (name: v: name == from) attrset) // {
+ ${to} = attrset.${from};
+ };
}
diff --git a/modules/blocks/authelia.nix b/modules/blocks/authelia.nix
index ac7d26a..3670c20 100644
--- a/modules/blocks/authelia.nix
+++ b/modules/blocks/authelia.nix
@@ -341,7 +341,7 @@ in
identity_providers.oidc.clients = clients;
};
resultPath = "/var/lib/authelia-${fqdn}/oidc_clients.yaml";
- generator = name: value: lib.generators.toYAML {} value;
+ generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toYAML {});
};
in
lib.mkBefore (mkCfg cfg.oidcClients);
diff --git a/modules/services/arr.nix b/modules/services/arr.nix
index 7b4200b..8b0db9d 100644
--- a/modules/services/arr.nix
+++ b/modules/services/arr.nix
@@ -8,7 +8,7 @@ let
apps = {
radarr = {
- settingsFormat = formatXML {};
+ settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for radarr.";
@@ -16,7 +16,7 @@ let
type = lib.types.submodule {
freeformType = apps.radarr.settingsFormat.type;
options = {
- APIKey = lib.mkOption {
+ ApiKey = lib.mkOption {
type = shblib.secretFileType;
description = "Path to api key secret file.";
};
@@ -66,7 +66,7 @@ let
};
};
sonarr = {
- settingsFormat = formatXML {};
+ settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for sonarr.";
@@ -74,7 +74,7 @@ let
type = lib.types.submodule {
freeformType = apps.sonarr.settingsFormat.type;
options = {
- APIKey = lib.mkOption {
+ ApiKey = lib.mkOption {
type = shblib.secretFileType;
description = "Path to api key secret file.";
};
@@ -119,7 +119,7 @@ let
};
};
bazarr = {
- settingsFormat = formatXML {};
+ settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for bazarr.";
@@ -144,7 +144,7 @@ let
};
};
readarr = {
- settingsFormat = formatXML {};
+ settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for readarr.";
@@ -168,7 +168,7 @@ let
};
};
lidarr = {
- settingsFormat = formatXML {};
+ settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for lidarr.";
@@ -200,7 +200,7 @@ let
type = lib.types.submodule {
freeformType = apps.jackett.settingsFormat.type;
options = {
- APIKey = lib.mkOption {
+ ApiKey = lib.mkOption {
type = shblib.secretFileType;
description = "Path to api key secret file.";
};
@@ -291,42 +291,6 @@ let
};
};
- formatXML = {}: {
- type = with lib.types; let
- valueType = nullOr (oneOf [
- bool
- int
- float
- str
- path
- (attrsOf valueType)
- (listOf valueType)
- ]) // {
- description = "XML value";
- };
- in valueType;
-
- generate = name: value: builtins.readFile (pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
- value = builtins.toJSON {Config = value;};
- passAsFile = [ "value" ];
- } (pkgs.writers.writePython3 "dict2xml" {
- libraries = with python3.pkgs; [ python dict2xml ];
- } ''
- import os
- import json
- from dict2xml import dict2xml
-
- with open(os.environ["valuePath"]) as f:
- content = json.loads(f.read())
- if content is None:
- print("Could not parse env var valuePath as json")
- os.exit(2)
- with open(os.environ["out"], "w") as out:
- out.write(dict2xml(content))
- '')) {});
-
- };
-
appOption = name: c: lib.nameValuePair name (lib.mkOption {
description = "Configuration for ${name}";
default = {};
@@ -402,7 +366,7 @@ in
AuthenticationMethod = "External";
});
resultPath = "${config.services.radarr.dataDir}/config.xml";
- generator = apps.radarr.settingsFormat.generate;
+ generator = shblib.replaceSecretsFormatAdapter apps.radarr.settingsFormat;
};
shb.nginx.autheliaProtect = [ (autheliaProtect {} cfg') ];
@@ -563,7 +527,7 @@ in
extraGroups = [ "media" ];
};
systemd.services.jackett.preStart = shblib.replaceSecrets {
- userConfig = cfg'.settings;
+ userConfig = shblib.renameAttrName cfg'.settings "ApiKey" "APIKey";
resultPath = "${config.services.jackett.dataDir}/ServerConfig.json";
generator = apps.jackett.settingsFormat.generate;
};
diff --git a/test/modules/arr.nix b/test/modules/arr.nix
index 2fbe58f..2ca7d39 100644
--- a/test/modules/arr.nix
+++ b/test/modules/arr.nix
@@ -126,7 +126,7 @@ in
enable = true;
authEndpoint = "https://oidc.example.com";
settings = {
- APIKey.source = pkgs.writeText "key" "/run/radarr/apikey";
+ ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey";
};
};
};
@@ -199,7 +199,7 @@ in
enable = true;
authEndpoint = "https://oidc.example.com";
settings = {
- APIKey.source = pkgs.writeText "key" "/run/radarr/apikey";
+ ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey";
};
backupCfg = {
enable = true;
diff --git a/test/modules/lib.nix b/test/modules/lib.nix
index a34dcf5..0a8227e 100644
--- a/test/modules/lib.nix
+++ b/test/modules/lib.nix
@@ -107,4 +107,22 @@ in
}
);
};
+
+ testParseXML = {
+ expected = {
+ "a" = {
+ "b" = "1";
+ "c" = {
+ "d" = "1";
+ };
+ };
+ };
+
+ expr = shblib.parseXML ''
+ <a>
+ <b>1</b>
+ <c><d>1</d></c>
+ </a>
+ '';
+ };
}
diff --git a/test/vm/arr.nix b/test/vm/arr.nix
index a14b75c..e4e8be4 100644
--- a/test/vm/arr.nix
+++ b/test/vm/arr.nix
@@ -2,9 +2,11 @@
let
pkgs' = pkgs;
# TODO: Test login
- commonTestScript = appname: { nodes, ... }:
+ commonTestScript = appname: cfgPathFn: { nodes, ... }:
let
shbapp = nodes.server.shb.arr.${appname};
+ cfgPath = cfgPathFn shbapp;
+ apiKey = if (shbapp.settings ? ApiKey) then "01234567890123456789" else null;
hasSSL = !(isNull shbapp.ssl);
fqdn = if hasSSL then "https://${appname}.example.com" else "http://${appname}.example.com";
healthUrl = "/health";
@@ -48,9 +50,15 @@ let
if response['code'] != 200:
raise Exception(f"Code is {response['code']}")
+ '' + lib.optionalString (apiKey != null) ''
+
+ with subtest("apikey"):
+ config = server.succeed("cat ${cfgPath}")
+ if "${apiKey}" not in config:
+ raise Exception(f"Unexpected API Key. Want '${apiKey}', got '{config}'")
'';
- basic = appname: pkgs.testers.runNixOSTest {
+ basic = appname: cfgPathFn: pkgs.testers.runNixOSTest {
name = "arr-${appname}-basic";
nodes.server = { config, pkgs, ... }: {
@@ -73,7 +81,7 @@ let
domain = "example.com";
subdomain = appname;
- settings.APIKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters.
+ settings.ApiKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters.
};
# Nginx port.
networking.firewall.allowedTCPPorts = [ 80 ];
@@ -81,14 +89,14 @@ let
nodes.client = {};
- testScript = commonTestScript appname;
+ testScript = commonTestScript appname cfgPathFn;
};
in
{
- radarr_basic = basic "radarr";
- sonarr_basic = basic "sonarr";
- bazarr_basic = basic "bazarr";
- readarr_basic = basic "readarr";
- lidarr_basic = basic "lidarr";
- jackett_basic = basic "jackett";
+ radarr_basic = basic "radarr" (cfg: "${cfg.dataDir}/config.xml");
+ sonarr_basic = basic "sonarr" (cfg: "${cfg.dataDir}/config.xml");
+ bazarr_basic = basic "bazarr" (cfg: "/var/lib/bazarr/config.xml");
+ readarr_basic = basic "readarr" (cfg: "${cfg.dataDir}/config.xml");
+ lidarr_basic = basic "lidarr" (cfg: "${cfg.dataDir}/config.xml");
+ jackett_basic = basic "jackett" (cfg: "${cfg.dataDir}/ServerConfig.json");
}
diff --git a/test/vm/lib.nix b/test/vm/lib.nix
index 681e89e..d594650 100644
--- a/test/vm/lib.nix
+++ b/test/vm/lib.nix
@@ -36,10 +36,22 @@ in
inherit replacements;
};
- replaceInTemplate2 = shblib.replaceSecrets {
+ replaceInTemplateJSON = shblib.replaceSecrets {
inherit userConfig;
- resultPath = "/var/lib/config2.yaml";
- generator = name: value: lib.generators.toJSON {} value;
+ resultPath = "/var/lib/config.json";
+ generator = shblib.replaceSecretsFormatAdapter (pkgs.formats.json {});
+ };
+
+ replaceInTemplateJSONGen = shblib.replaceSecrets {
+ inherit userConfig;
+ resultPath = "/var/lib/config_gen.json";
+ generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toJSON {});
+ };
+
+ replaceInTemplateXML = shblib.replaceSecrets {
+ inherit userConfig;
+ resultPath = "/var/lib/config.xml";
+ generator = shblib.replaceSecretsFormatAdapter (shblib.formatXML {enclosingRoot = "Root";});
};
in
pkgs.testers.runNixOSTest {
@@ -60,27 +72,72 @@ in
system.activationScripts = {
libtest = replaceInTemplate;
- libtest2 = replaceInTemplate2;
+ libtestJSON = replaceInTemplateJSON;
+ libtestJSONGen = replaceInTemplateJSONGen;
+ libtestXML = replaceInTemplateXML;
};
};
testScript = { nodes, ... }: ''
import json
+ from collections import ChainMap
+ from xml.etree import ElementTree
+
start_all()
+ machine.wait_for_file("/var/lib/config.yaml")
+ machine.wait_for_file("/var/lib/config.json")
+ machine.wait_for_file("/var/lib/config_gen.json")
+ machine.wait_for_file("/var/lib/config.xml")
+
+ def xml_to_dict_recursive(root):
+ all_descendants = list(root)
+ if len(all_descendants) == 0:
+ return {root.tag: root.text}
+ else:
+ merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
+ return {root.tag: dict(merged_dict)}
wantedConfig = json.loads('${lib.generators.toJSON {} wantedConfig}')
- gotConfig = json.loads(machine.succeed("cat /var/lib/config.yaml"))
- gotConfig2 = json.loads(machine.succeed("cat /var/lib/config2.yaml"))
- # For debugging purpose
- print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
- print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate2" replaceInTemplate2}"))
+ with subtest("config"):
+ print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
- if wantedConfig != gotConfig:
- raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+ gotConfig = machine.succeed("cat /var/lib/config.yaml")
+ print(gotConfig)
+ gotConfig = json.loads(gotConfig)
- if wantedConfig != gotConfig2:
- raise Exception("\nwantedConfig: {}\n!= gotConfig2: {}".format(wantedConfig, gotConfig))
+ if wantedConfig != gotConfig:
+ raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+
+ with subtest("config JSON Gen"):
+ print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSONGen" replaceInTemplateJSONGen}"))
+
+ gotConfig = machine.succeed("cat /var/lib/config_gen.json")
+ print(gotConfig)
+ gotConfig = json.loads(gotConfig)
+
+ if wantedConfig != gotConfig:
+ raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+
+ with subtest("config JSON"):
+ print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSON" replaceInTemplateJSON}"))
+
+ gotConfig = machine.succeed("cat /var/lib/config.json")
+ print(gotConfig)
+ gotConfig = json.loads(gotConfig)
+
+ if wantedConfig != gotConfig:
+ raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+
+ with subtest("config XML"):
+ print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateXML" replaceInTemplateXML}"))
+
+ gotConfig = machine.succeed("cat /var/lib/config.xml")
+ print(gotConfig)
+ gotConfig = xml_to_dict_recursive(ElementTree.XML(gotConfig))['Root']
+
+ if wantedConfig != gotConfig:
+ raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
'';
};
}