make config with secrets correctly generated
This commit is contained in:
parent
dc46ec8eda
commit
8ec12338fd
7 changed files with 214 additions and 78 deletions
101
lib/default.nix
101
lib/default.nix
|
@ -1,10 +1,15 @@
|
|||
{ pkgs, lib }:
|
||||
rec {
|
||||
# Replace secrets in a file.
|
||||
# - userConfig is an attrset that will produce a config file.
|
||||
# - resultPath is the location the config file should have on the filesystem.
|
||||
# - generator is a function taking two arguments name and value and returning path in the nix
|
||||
# nix store where the
|
||||
replaceSecrets = { userConfig, resultPath, generator }:
|
||||
let
|
||||
configWithTemplates = withReplacements userConfig;
|
||||
|
||||
nonSecretConfigFile = pkgs.writeText "${resultPath}.template" (generator "template" configWithTemplates);
|
||||
nonSecretConfigFile = generator "template" configWithTemplates;
|
||||
|
||||
replacements = getReplacements userConfig;
|
||||
in
|
||||
|
@ -13,6 +18,9 @@ rec {
|
|||
inherit resultPath replacements;
|
||||
};
|
||||
|
||||
replaceSecretsFormatAdapter = format: format.generate;
|
||||
replaceSecretsGeneratorAdapter = generator: name: value: pkgs.writeText "generator " (generator value);
|
||||
|
||||
template = file: newPath: replacements: replaceSecretsScript {
|
||||
inherit file replacements;
|
||||
resultPath = newPath;
|
||||
|
@ -22,6 +30,9 @@ rec {
|
|||
let
|
||||
templatePath = resultPath + ".template";
|
||||
sedPatterns = lib.strings.concatStringsSep " " (lib.attrsets.mapAttrsToList (from: to: "-e \"s|${from}|${to}|\"") replacements);
|
||||
sedCmd = if replacements == {}
|
||||
then "cat"
|
||||
else "${pkgs.gnused}/bin/sed ${sedPatterns}";
|
||||
in
|
||||
''
|
||||
set -euo pipefail
|
||||
|
@ -29,11 +40,7 @@ rec {
|
|||
mkdir -p $(dirname ${templatePath})
|
||||
ln -fs ${file} ${templatePath}
|
||||
rm -f ${resultPath}
|
||||
if [ -z "${sedPatterns}" ]; then
|
||||
cat ${templatePath} > ${resultPath}
|
||||
else
|
||||
${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath} > ${resultPath}
|
||||
fi
|
||||
${sedCmd} ${templatePath} > ${resultPath}
|
||||
'';
|
||||
|
||||
secretFileType = lib.types.submodule {
|
||||
|
@ -115,4 +122,86 @@ rec {
|
|||
lib.lists.concatMap (collect pred) attrs
|
||||
else
|
||||
[];
|
||||
|
||||
# Generator for XML
|
||||
formatXML = {
|
||||
enclosingRoot ? null
|
||||
}: {
|
||||
type = with lib.types; let
|
||||
valueType = nullOr (oneOf [
|
||||
bool
|
||||
int
|
||||
float
|
||||
str
|
||||
path
|
||||
(attrsOf valueType)
|
||||
(listOf valueType)
|
||||
]) // {
|
||||
description = "XML value";
|
||||
};
|
||||
in valueType;
|
||||
|
||||
generate = name: value: pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
|
||||
value = builtins.toJSON (
|
||||
if enclosingRoot == null then
|
||||
value
|
||||
else
|
||||
{ ${enclosingRoot} = value; });
|
||||
passAsFile = [ "value" ];
|
||||
} (pkgs.writers.writePython3 "dict2xml" {
|
||||
libraries = with python3.pkgs; [ python dict2xml ];
|
||||
} ''
|
||||
import os
|
||||
import json
|
||||
from dict2xml import dict2xml
|
||||
|
||||
with open(os.environ["valuePath"]) as f:
|
||||
content = json.loads(f.read())
|
||||
if content is None:
|
||||
print("Could not parse env var valuePath as json")
|
||||
os.exit(2)
|
||||
with open(os.environ["out"], "w") as out:
|
||||
out.write(dict2xml(content))
|
||||
'')) {};
|
||||
|
||||
};
|
||||
|
||||
parseXML = xml:
|
||||
let
|
||||
xmlToJsonFile = pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
|
||||
inherit xml;
|
||||
passAsFile = [ "xml" ];
|
||||
} (pkgs.writers.writePython3 "xml2json" {
|
||||
libraries = with python3.pkgs; [ python ];
|
||||
} ''
|
||||
import os
|
||||
import json
|
||||
from collections import ChainMap
|
||||
from xml.etree import ElementTree
|
||||
|
||||
|
||||
def xml_to_dict_recursive(root):
|
||||
all_descendants = list(root)
|
||||
if len(all_descendants) == 0:
|
||||
return {root.tag: root.text}
|
||||
else:
|
||||
merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
|
||||
return {root.tag: dict(merged_dict)}
|
||||
|
||||
|
||||
with open(os.environ["xmlPath"]) as f:
|
||||
root = ElementTree.XML(f.read())
|
||||
xml = xml_to_dict_recursive(root)
|
||||
j = json.dumps(xml)
|
||||
|
||||
with open(os.environ["out"], "w") as out:
|
||||
out.write(j)
|
||||
'')) {};
|
||||
in
|
||||
builtins.fromJSON (builtins.readFile xmlToJsonFile);
|
||||
|
||||
renameAttrName = attrset: from: to:
|
||||
(lib.attrsets.filterAttrs (name: v: name == from) attrset) // {
|
||||
${to} = attrset.${from};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -341,7 +341,7 @@ in
|
|||
identity_providers.oidc.clients = clients;
|
||||
};
|
||||
resultPath = "/var/lib/authelia-${fqdn}/oidc_clients.yaml";
|
||||
generator = name: value: lib.generators.toYAML {} value;
|
||||
generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toYAML {});
|
||||
};
|
||||
in
|
||||
lib.mkBefore (mkCfg cfg.oidcClients);
|
||||
|
|
|
@ -8,7 +8,7 @@ let
|
|||
|
||||
apps = {
|
||||
radarr = {
|
||||
settingsFormat = formatXML {};
|
||||
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
|
||||
moreOptions = {
|
||||
settings = lib.mkOption {
|
||||
description = "Specific options for radarr.";
|
||||
|
@ -16,7 +16,7 @@ let
|
|||
type = lib.types.submodule {
|
||||
freeformType = apps.radarr.settingsFormat.type;
|
||||
options = {
|
||||
APIKey = lib.mkOption {
|
||||
ApiKey = lib.mkOption {
|
||||
type = shblib.secretFileType;
|
||||
description = "Path to api key secret file.";
|
||||
};
|
||||
|
@ -66,7 +66,7 @@ let
|
|||
};
|
||||
};
|
||||
sonarr = {
|
||||
settingsFormat = formatXML {};
|
||||
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
|
||||
moreOptions = {
|
||||
settings = lib.mkOption {
|
||||
description = "Specific options for sonarr.";
|
||||
|
@ -74,7 +74,7 @@ let
|
|||
type = lib.types.submodule {
|
||||
freeformType = apps.sonarr.settingsFormat.type;
|
||||
options = {
|
||||
APIKey = lib.mkOption {
|
||||
ApiKey = lib.mkOption {
|
||||
type = shblib.secretFileType;
|
||||
description = "Path to api key secret file.";
|
||||
};
|
||||
|
@ -119,7 +119,7 @@ let
|
|||
};
|
||||
};
|
||||
bazarr = {
|
||||
settingsFormat = formatXML {};
|
||||
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
|
||||
moreOptions = {
|
||||
settings = lib.mkOption {
|
||||
description = "Specific options for bazarr.";
|
||||
|
@ -144,7 +144,7 @@ let
|
|||
};
|
||||
};
|
||||
readarr = {
|
||||
settingsFormat = formatXML {};
|
||||
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
|
||||
moreOptions = {
|
||||
settings = lib.mkOption {
|
||||
description = "Specific options for readarr.";
|
||||
|
@ -168,7 +168,7 @@ let
|
|||
};
|
||||
};
|
||||
lidarr = {
|
||||
settingsFormat = formatXML {};
|
||||
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
|
||||
moreOptions = {
|
||||
settings = lib.mkOption {
|
||||
description = "Specific options for lidarr.";
|
||||
|
@ -200,7 +200,7 @@ let
|
|||
type = lib.types.submodule {
|
||||
freeformType = apps.jackett.settingsFormat.type;
|
||||
options = {
|
||||
APIKey = lib.mkOption {
|
||||
ApiKey = lib.mkOption {
|
||||
type = shblib.secretFileType;
|
||||
description = "Path to api key secret file.";
|
||||
};
|
||||
|
@ -291,42 +291,6 @@ let
|
|||
};
|
||||
};
|
||||
|
||||
formatXML = {}: {
|
||||
type = with lib.types; let
|
||||
valueType = nullOr (oneOf [
|
||||
bool
|
||||
int
|
||||
float
|
||||
str
|
||||
path
|
||||
(attrsOf valueType)
|
||||
(listOf valueType)
|
||||
]) // {
|
||||
description = "XML value";
|
||||
};
|
||||
in valueType;
|
||||
|
||||
generate = name: value: builtins.readFile (pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
|
||||
value = builtins.toJSON {Config = value;};
|
||||
passAsFile = [ "value" ];
|
||||
} (pkgs.writers.writePython3 "dict2xml" {
|
||||
libraries = with python3.pkgs; [ python dict2xml ];
|
||||
} ''
|
||||
import os
|
||||
import json
|
||||
from dict2xml import dict2xml
|
||||
|
||||
with open(os.environ["valuePath"]) as f:
|
||||
content = json.loads(f.read())
|
||||
if content is None:
|
||||
print("Could not parse env var valuePath as json")
|
||||
os.exit(2)
|
||||
with open(os.environ["out"], "w") as out:
|
||||
out.write(dict2xml(content))
|
||||
'')) {});
|
||||
|
||||
};
|
||||
|
||||
appOption = name: c: lib.nameValuePair name (lib.mkOption {
|
||||
description = "Configuration for ${name}";
|
||||
default = {};
|
||||
|
@ -402,7 +366,7 @@ in
|
|||
AuthenticationMethod = "External";
|
||||
});
|
||||
resultPath = "${config.services.radarr.dataDir}/config.xml";
|
||||
generator = apps.radarr.settingsFormat.generate;
|
||||
generator = shblib.replaceSecretsFormatAdapter apps.radarr.settingsFormat;
|
||||
};
|
||||
|
||||
shb.nginx.autheliaProtect = [ (autheliaProtect {} cfg') ];
|
||||
|
@ -563,7 +527,7 @@ in
|
|||
extraGroups = [ "media" ];
|
||||
};
|
||||
systemd.services.jackett.preStart = shblib.replaceSecrets {
|
||||
userConfig = cfg'.settings;
|
||||
userConfig = shblib.renameAttrName cfg'.settings "ApiKey" "APIKey";
|
||||
resultPath = "${config.services.jackett.dataDir}/ServerConfig.json";
|
||||
generator = apps.jackett.settingsFormat.generate;
|
||||
};
|
||||
|
|
|
@ -126,7 +126,7 @@ in
|
|||
enable = true;
|
||||
authEndpoint = "https://oidc.example.com";
|
||||
settings = {
|
||||
APIKey.source = pkgs.writeText "key" "/run/radarr/apikey";
|
||||
ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
@ -199,7 +199,7 @@ in
|
|||
enable = true;
|
||||
authEndpoint = "https://oidc.example.com";
|
||||
settings = {
|
||||
APIKey.source = pkgs.writeText "key" "/run/radarr/apikey";
|
||||
ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey";
|
||||
};
|
||||
backupCfg = {
|
||||
enable = true;
|
||||
|
|
|
@ -107,4 +107,22 @@ in
|
|||
}
|
||||
);
|
||||
};
|
||||
|
||||
testParseXML = {
|
||||
expected = {
|
||||
"a" = {
|
||||
"b" = "1";
|
||||
"c" = {
|
||||
"d" = "1";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
expr = shblib.parseXML ''
|
||||
<a>
|
||||
<b>1</b>
|
||||
<c><d>1</d></c>
|
||||
</a>
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
|
@ -2,9 +2,11 @@
|
|||
let
|
||||
pkgs' = pkgs;
|
||||
# TODO: Test login
|
||||
commonTestScript = appname: { nodes, ... }:
|
||||
commonTestScript = appname: cfgPathFn: { nodes, ... }:
|
||||
let
|
||||
shbapp = nodes.server.shb.arr.${appname};
|
||||
cfgPath = cfgPathFn shbapp;
|
||||
apiKey = if (shbapp.settings ? ApiKey) then "01234567890123456789" else null;
|
||||
hasSSL = !(isNull shbapp.ssl);
|
||||
fqdn = if hasSSL then "https://${appname}.example.com" else "http://${appname}.example.com";
|
||||
healthUrl = "/health";
|
||||
|
@ -48,9 +50,15 @@ let
|
|||
|
||||
if response['code'] != 200:
|
||||
raise Exception(f"Code is {response['code']}")
|
||||
'' + lib.optionalString (apiKey != null) ''
|
||||
|
||||
with subtest("apikey"):
|
||||
config = server.succeed("cat ${cfgPath}")
|
||||
if "${apiKey}" not in config:
|
||||
raise Exception(f"Unexpected API Key. Want '${apiKey}', got '{config}'")
|
||||
'';
|
||||
|
||||
basic = appname: pkgs.testers.runNixOSTest {
|
||||
basic = appname: cfgPathFn: pkgs.testers.runNixOSTest {
|
||||
name = "arr-${appname}-basic";
|
||||
|
||||
nodes.server = { config, pkgs, ... }: {
|
||||
|
@ -73,7 +81,7 @@ let
|
|||
domain = "example.com";
|
||||
subdomain = appname;
|
||||
|
||||
settings.APIKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters.
|
||||
settings.ApiKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters.
|
||||
};
|
||||
# Nginx port.
|
||||
networking.firewall.allowedTCPPorts = [ 80 ];
|
||||
|
@ -81,14 +89,14 @@ let
|
|||
|
||||
nodes.client = {};
|
||||
|
||||
testScript = commonTestScript appname;
|
||||
testScript = commonTestScript appname cfgPathFn;
|
||||
};
|
||||
in
|
||||
{
|
||||
radarr_basic = basic "radarr";
|
||||
sonarr_basic = basic "sonarr";
|
||||
bazarr_basic = basic "bazarr";
|
||||
readarr_basic = basic "readarr";
|
||||
lidarr_basic = basic "lidarr";
|
||||
jackett_basic = basic "jackett";
|
||||
radarr_basic = basic "radarr" (cfg: "${cfg.dataDir}/config.xml");
|
||||
sonarr_basic = basic "sonarr" (cfg: "${cfg.dataDir}/config.xml");
|
||||
bazarr_basic = basic "bazarr" (cfg: "/var/lib/bazarr/config.xml");
|
||||
readarr_basic = basic "readarr" (cfg: "${cfg.dataDir}/config.xml");
|
||||
lidarr_basic = basic "lidarr" (cfg: "${cfg.dataDir}/config.xml");
|
||||
jackett_basic = basic "jackett" (cfg: "${cfg.dataDir}/ServerConfig.json");
|
||||
}
|
||||
|
|
|
@ -36,10 +36,22 @@ in
|
|||
inherit replacements;
|
||||
};
|
||||
|
||||
replaceInTemplate2 = shblib.replaceSecrets {
|
||||
replaceInTemplateJSON = shblib.replaceSecrets {
|
||||
inherit userConfig;
|
||||
resultPath = "/var/lib/config2.yaml";
|
||||
generator = name: value: lib.generators.toJSON {} value;
|
||||
resultPath = "/var/lib/config.json";
|
||||
generator = shblib.replaceSecretsFormatAdapter (pkgs.formats.json {});
|
||||
};
|
||||
|
||||
replaceInTemplateJSONGen = shblib.replaceSecrets {
|
||||
inherit userConfig;
|
||||
resultPath = "/var/lib/config_gen.json";
|
||||
generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toJSON {});
|
||||
};
|
||||
|
||||
replaceInTemplateXML = shblib.replaceSecrets {
|
||||
inherit userConfig;
|
||||
resultPath = "/var/lib/config.xml";
|
||||
generator = shblib.replaceSecretsFormatAdapter (shblib.formatXML {enclosingRoot = "Root";});
|
||||
};
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
|
@ -60,27 +72,72 @@ in
|
|||
|
||||
system.activationScripts = {
|
||||
libtest = replaceInTemplate;
|
||||
libtest2 = replaceInTemplate2;
|
||||
libtestJSON = replaceInTemplateJSON;
|
||||
libtestJSONGen = replaceInTemplateJSONGen;
|
||||
libtestXML = replaceInTemplateXML;
|
||||
};
|
||||
};
|
||||
|
||||
testScript = { nodes, ... }: ''
|
||||
import json
|
||||
from collections import ChainMap
|
||||
from xml.etree import ElementTree
|
||||
|
||||
start_all()
|
||||
machine.wait_for_file("/var/lib/config.yaml")
|
||||
machine.wait_for_file("/var/lib/config.json")
|
||||
machine.wait_for_file("/var/lib/config_gen.json")
|
||||
machine.wait_for_file("/var/lib/config.xml")
|
||||
|
||||
def xml_to_dict_recursive(root):
|
||||
all_descendants = list(root)
|
||||
if len(all_descendants) == 0:
|
||||
return {root.tag: root.text}
|
||||
else:
|
||||
merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
|
||||
return {root.tag: dict(merged_dict)}
|
||||
|
||||
wantedConfig = json.loads('${lib.generators.toJSON {} wantedConfig}')
|
||||
gotConfig = json.loads(machine.succeed("cat /var/lib/config.yaml"))
|
||||
gotConfig2 = json.loads(machine.succeed("cat /var/lib/config2.yaml"))
|
||||
|
||||
# For debugging purpose
|
||||
with subtest("config"):
|
||||
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
|
||||
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate2" replaceInTemplate2}"))
|
||||
|
||||
gotConfig = machine.succeed("cat /var/lib/config.yaml")
|
||||
print(gotConfig)
|
||||
gotConfig = json.loads(gotConfig)
|
||||
|
||||
if wantedConfig != gotConfig:
|
||||
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
|
||||
|
||||
if wantedConfig != gotConfig2:
|
||||
raise Exception("\nwantedConfig: {}\n!= gotConfig2: {}".format(wantedConfig, gotConfig))
|
||||
with subtest("config JSON Gen"):
|
||||
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSONGen" replaceInTemplateJSONGen}"))
|
||||
|
||||
gotConfig = machine.succeed("cat /var/lib/config_gen.json")
|
||||
print(gotConfig)
|
||||
gotConfig = json.loads(gotConfig)
|
||||
|
||||
if wantedConfig != gotConfig:
|
||||
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
|
||||
|
||||
with subtest("config JSON"):
|
||||
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSON" replaceInTemplateJSON}"))
|
||||
|
||||
gotConfig = machine.succeed("cat /var/lib/config.json")
|
||||
print(gotConfig)
|
||||
gotConfig = json.loads(gotConfig)
|
||||
|
||||
if wantedConfig != gotConfig:
|
||||
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
|
||||
|
||||
with subtest("config XML"):
|
||||
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateXML" replaceInTemplateXML}"))
|
||||
|
||||
gotConfig = machine.succeed("cat /var/lib/config.xml")
|
||||
print(gotConfig)
|
||||
gotConfig = xml_to_dict_recursive(ElementTree.XML(gotConfig))['Root']
|
||||
|
||||
if wantedConfig != gotConfig:
|
||||
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue