hosting/selfhostblocks
1
0
Fork 0
Code Issues Activity

make config with secrets correctly generated

Browse source
This commit is contained in:
ibizaman 2024-05-23 14:28:08 -07:00 committed by Pierre Penninckx
parent dc46ec8eda
commit 8ec12338fd
7 changed files with 214 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

101
lib/default.nix
View file

@ -1,10 +1,15 @@
{ pkgs, lib }:
rec {
# Replace secrets in a file.
# - userConfig is an attrset that will produce a config file.
# - resultPath is the location the config file should have on the filesystem.
# - generator is a function taking two arguments name and value and returning path in the nix
# nix store where the
replaceSecrets = { userConfig, resultPath, generator }:
let
configWithTemplates = withReplacements userConfig;
nonSecretConfigFile = pkgs.writeText "${resultPath}.template" (generator "template" configWithTemplates);
nonSecretConfigFile = generator "template" configWithTemplates;
replacements = getReplacements userConfig;
in
@ -13,6 +18,9 @@ rec {
inherit resultPath replacements;
};
replaceSecretsFormatAdapter = format: format.generate;
replaceSecretsGeneratorAdapter = generator: name: value: pkgs.writeText "generator " (generator value);
template = file: newPath: replacements: replaceSecretsScript {
inherit file replacements;
resultPath = newPath;
@ -22,6 +30,9 @@ rec {
let
templatePath = resultPath + ".template";
sedPatterns = lib.strings.concatStringsSep " " (lib.attrsets.mapAttrsToList (from: to: "-e \"s|${from}|${to}|\"") replacements);
sedCmd = if replacements == {}
then "cat"
else "${pkgs.gnused}/bin/sed ${sedPatterns}";
in
''
set -euo pipefail
@ -29,11 +40,7 @@ rec {
mkdir -p $(dirname ${templatePath})
ln -fs ${file} ${templatePath}
rm -f ${resultPath}
if [ -z "${sedPatterns}" ]; then
cat ${templatePath} > ${resultPath}
else
${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath} > ${resultPath}
fi
${sedCmd} ${templatePath} > ${resultPath}
'';
secretFileType = lib.types.submodule {
@ -115,4 +122,86 @@ rec {
lib.lists.concatMap (collect pred) attrs
else
[];
# Generator for XML
formatXML = {
enclosingRoot ? null
}: {
type = with lib.types; let
valueType = nullOr (oneOf [
bool
int
float
str
path
(attrsOf valueType)
(listOf valueType)
]) // {
description = "XML value";
};
in valueType;
generate = name: value: pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
value = builtins.toJSON (
if enclosingRoot == null then
value
else
{ ${enclosingRoot} = value; });
passAsFile = [ "value" ];
} (pkgs.writers.writePython3 "dict2xml" {
libraries = with python3.pkgs; [ python dict2xml ];
} ''
import os
import json
from dict2xml import dict2xml
with open(os.environ["valuePath"]) as f:
content = json.loads(f.read())
if content is None:
print("Could not parse env var valuePath as json")
os.exit(2)
with open(os.environ["out"], "w") as out:
out.write(dict2xml(content))
'')) {};
};
parseXML = xml:
let
xmlToJsonFile = pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
inherit xml;
passAsFile = [ "xml" ];
} (pkgs.writers.writePython3 "xml2json" {
libraries = with python3.pkgs; [ python ];
} ''
import os
import json
from collections import ChainMap
from xml.etree import ElementTree
def xml_to_dict_recursive(root):
all_descendants = list(root)
if len(all_descendants) == 0:
return {root.tag: root.text}
else:
merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
return {root.tag: dict(merged_dict)}
with open(os.environ["xmlPath"]) as f:
root = ElementTree.XML(f.read())
xml = xml_to_dict_recursive(root)
j = json.dumps(xml)
with open(os.environ["out"], "w") as out:
out.write(j)
'')) {};
in
builtins.fromJSON (builtins.readFile xmlToJsonFile);
renameAttrName = attrset: from: to:
(lib.attrsets.filterAttrs (name: v: name == from) attrset) // {
${to} = attrset.${from};
};
}

2
modules/blocks/authelia.nix
View file

@ -341,7 +341,7 @@ in
identity_providers.oidc.clients = clients;
};
resultPath = "/var/lib/authelia-${fqdn}/oidc_clients.yaml";
generator = name: value: lib.generators.toYAML {} value;
generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toYAML {});
};
in
lib.mkBefore (mkCfg cfg.oidcClients);

56
modules/services/arr.nix
View file

@ -8,7 +8,7 @@ let
apps = {
radarr = {
settingsFormat = formatXML {};
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for radarr.";
@ -16,7 +16,7 @@ let
type = lib.types.submodule {
freeformType = apps.radarr.settingsFormat.type;
options = {
APIKey = lib.mkOption {
ApiKey = lib.mkOption {
type = shblib.secretFileType;
description = "Path to api key secret file.";
};
@ -66,7 +66,7 @@ let
};
};
sonarr = {
settingsFormat = formatXML {};
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for sonarr.";
@ -74,7 +74,7 @@ let
type = lib.types.submodule {
freeformType = apps.sonarr.settingsFormat.type;
options = {
APIKey = lib.mkOption {
ApiKey = lib.mkOption {
type = shblib.secretFileType;
description = "Path to api key secret file.";
};
@ -119,7 +119,7 @@ let
};
};
bazarr = {
settingsFormat = formatXML {};
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for bazarr.";
@ -144,7 +144,7 @@ let
};
};
readarr = {
settingsFormat = formatXML {};
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for readarr.";
@ -168,7 +168,7 @@ let
};
};
lidarr = {
settingsFormat = formatXML {};
settingsFormat = shblib.formatXML { enclosingRoot = "Config"; };
moreOptions = {
settings = lib.mkOption {
description = "Specific options for lidarr.";
@ -200,7 +200,7 @@ let
type = lib.types.submodule {
freeformType = apps.jackett.settingsFormat.type;
options = {
APIKey = lib.mkOption {
ApiKey = lib.mkOption {
type = shblib.secretFileType;
description = "Path to api key secret file.";
};
@ -291,42 +291,6 @@ let
};
};
formatXML = {}: {
type = with lib.types; let
valueType = nullOr (oneOf [
bool
int
float
str
path
(attrsOf valueType)
(listOf valueType)
]) // {
description = "XML value";
};
in valueType;
generate = name: value: builtins.readFile (pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
value = builtins.toJSON {Config = value;};
passAsFile = [ "value" ];
} (pkgs.writers.writePython3 "dict2xml" {
libraries = with python3.pkgs; [ python dict2xml ];
} ''
import os
import json
from dict2xml import dict2xml
with open(os.environ["valuePath"]) as f:
content = json.loads(f.read())
if content is None:
print("Could not parse env var valuePath as json")
os.exit(2)
with open(os.environ["out"], "w") as out:
out.write(dict2xml(content))
'')) {});
};
appOption = name: c: lib.nameValuePair name (lib.mkOption {
description = "Configuration for ${name}";
default = {};
@ -402,7 +366,7 @@ in
AuthenticationMethod = "External";
});
resultPath = "${config.services.radarr.dataDir}/config.xml";
generator = apps.radarr.settingsFormat.generate;
generator = shblib.replaceSecretsFormatAdapter apps.radarr.settingsFormat;
};
shb.nginx.autheliaProtect = [ (autheliaProtect {} cfg') ];
@ -563,7 +527,7 @@ in
extraGroups = [ "media" ];
};
systemd.services.jackett.preStart = shblib.replaceSecrets {
userConfig = cfg'.settings;
userConfig = shblib.renameAttrName cfg'.settings "ApiKey" "APIKey";
resultPath = "${config.services.jackett.dataDir}/ServerConfig.json";
generator = apps.jackett.settingsFormat.generate;
};

4
test/modules/arr.nix
View file

@ -126,7 +126,7 @@ in
enable = true;
authEndpoint = "https://oidc.example.com";
settings = {
APIKey.source = pkgs.writeText "key" "/run/radarr/apikey";
ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey";
};
};
};
@ -199,7 +199,7 @@ in
enable = true;
authEndpoint = "https://oidc.example.com";
settings = {
APIKey.source = pkgs.writeText "key" "/run/radarr/apikey";
ApiKey.source = pkgs.writeText "key" "/run/radarr/apikey";
};
backupCfg = {
enable = true;

18
test/modules/lib.nix
View file

@ -107,4 +107,22 @@ in
}
);
};
testParseXML = {
expected = {
"a" = {
"b" = "1";
"c" = {
"d" = "1";
};
};
};
expr = shblib.parseXML ''
<a>
<b>1</b>
<c><d>1</d></c>
</a>
'';
};
}

28
test/vm/arr.nix
View file

@ -2,9 +2,11 @@
let
pkgs' = pkgs;
# TODO: Test login
commonTestScript = appname: { nodes, ... }:
commonTestScript = appname: cfgPathFn: { nodes, ... }:
let
shbapp = nodes.server.shb.arr.${appname};
cfgPath = cfgPathFn shbapp;
apiKey = if (shbapp.settings ? ApiKey) then "01234567890123456789" else null;
hasSSL = !(isNull shbapp.ssl);
fqdn = if hasSSL then "https://${appname}.example.com" else "http://${appname}.example.com";
healthUrl = "/health";
@ -48,9 +50,15 @@ let
if response['code'] != 200:
raise Exception(f"Code is {response['code']}")
'' + lib.optionalString (apiKey != null) ''
with subtest("apikey"):
config = server.succeed("cat ${cfgPath}")
if "${apiKey}" not in config:
raise Exception(f"Unexpected API Key. Want '${apiKey}', got '{config}'")
'';
basic = appname: pkgs.testers.runNixOSTest {
basic = appname: cfgPathFn: pkgs.testers.runNixOSTest {
name = "arr-${appname}-basic";
nodes.server = { config, pkgs, ... }: {
@ -73,7 +81,7 @@ let
domain = "example.com";
subdomain = appname;
settings.APIKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters.
settings.ApiKey.source = pkgs.writeText "APIKey" "01234567890123456789"; # Needs to be >=20 characters.
};
# Nginx port.
networking.firewall.allowedTCPPorts = [ 80 ];
@ -81,14 +89,14 @@ let
nodes.client = {};
testScript = commonTestScript appname;
testScript = commonTestScript appname cfgPathFn;
};
in
{
radarr_basic = basic "radarr";
sonarr_basic = basic "sonarr";
bazarr_basic = basic "bazarr";
readarr_basic = basic "readarr";
lidarr_basic = basic "lidarr";
jackett_basic = basic "jackett";
radarr_basic = basic "radarr" (cfg: "${cfg.dataDir}/config.xml");
sonarr_basic = basic "sonarr" (cfg: "${cfg.dataDir}/config.xml");
bazarr_basic = basic "bazarr" (cfg: "/var/lib/bazarr/config.xml");
readarr_basic = basic "readarr" (cfg: "${cfg.dataDir}/config.xml");
lidarr_basic = basic "lidarr" (cfg: "${cfg.dataDir}/config.xml");
jackett_basic = basic "jackett" (cfg: "${cfg.dataDir}/ServerConfig.json");
}

83
test/vm/lib.nix
View file

@ -36,10 +36,22 @@ in
inherit replacements;
};
replaceInTemplate2 = shblib.replaceSecrets {
replaceInTemplateJSON = shblib.replaceSecrets {
inherit userConfig;
resultPath = "/var/lib/config2.yaml";
generator = name: value: lib.generators.toJSON {} value;
resultPath = "/var/lib/config.json";
generator = shblib.replaceSecretsFormatAdapter (pkgs.formats.json {});
};
replaceInTemplateJSONGen = shblib.replaceSecrets {
inherit userConfig;
resultPath = "/var/lib/config_gen.json";
generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toJSON {});
};
replaceInTemplateXML = shblib.replaceSecrets {
inherit userConfig;
resultPath = "/var/lib/config.xml";
generator = shblib.replaceSecretsFormatAdapter (shblib.formatXML {enclosingRoot = "Root";});
};
in
pkgs.testers.runNixOSTest {
@ -60,27 +72,72 @@ in
system.activationScripts = {
libtest = replaceInTemplate;
libtest2 = replaceInTemplate2;
libtestJSON = replaceInTemplateJSON;
libtestJSONGen = replaceInTemplateJSONGen;
libtestXML = replaceInTemplateXML;
};
};
testScript = { nodes, ... }: ''
import json
from collections import ChainMap
from xml.etree import ElementTree
start_all()
machine.wait_for_file("/var/lib/config.yaml")
machine.wait_for_file("/var/lib/config.json")
machine.wait_for_file("/var/lib/config_gen.json")
machine.wait_for_file("/var/lib/config.xml")
def xml_to_dict_recursive(root):
all_descendants = list(root)
if len(all_descendants) == 0:
return {root.tag: root.text}
else:
merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
return {root.tag: dict(merged_dict)}
wantedConfig = json.loads('${lib.generators.toJSON {} wantedConfig}')
gotConfig = json.loads(machine.succeed("cat /var/lib/config.yaml"))
gotConfig2 = json.loads(machine.succeed("cat /var/lib/config2.yaml"))
# For debugging purpose
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate2" replaceInTemplate2}"))
with subtest("config"):
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
if wantedConfig != gotConfig:
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
gotConfig = machine.succeed("cat /var/lib/config.yaml")
print(gotConfig)
gotConfig = json.loads(gotConfig)
if wantedConfig != gotConfig2:
raise Exception("\nwantedConfig: {}\n!= gotConfig2: {}".format(wantedConfig, gotConfig))
if wantedConfig != gotConfig:
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
with subtest("config JSON Gen"):
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSONGen" replaceInTemplateJSONGen}"))
gotConfig = machine.succeed("cat /var/lib/config_gen.json")
print(gotConfig)
gotConfig = json.loads(gotConfig)
if wantedConfig != gotConfig:
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
with subtest("config JSON"):
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateJSON" replaceInTemplateJSON}"))
gotConfig = machine.succeed("cat /var/lib/config.json")
print(gotConfig)
gotConfig = json.loads(gotConfig)
if wantedConfig != gotConfig:
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
with subtest("config XML"):
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplateXML" replaceInTemplateXML}"))
gotConfig = machine.succeed("cat /var/lib/config.xml")
print(gotConfig)
gotConfig = xml_to_dict_recursive(ElementTree.XML(gotConfig))['Root']
if wantedConfig != gotConfig:
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
'';
};
}