1
0
Fork 0

move templating code to lib file

This commit is contained in:
ibizaman 2024-02-29 15:34:53 -08:00 committed by Pierre Penninckx
parent 6cf83e737e
commit fa206d0e15
10 changed files with 439 additions and 58 deletions

View file

@ -88,13 +88,20 @@
mergeTests (importFiles [
./test/modules/arr.nix
./test/modules/davfs.nix
./test/modules/lib.nix
./test/modules/nginx.nix
./test/modules/postgresql.nix
]);
};
lib = nix-flake-tests.lib.check {
inherit pkgs;
tests = pkgs.callPackage ./test/modules/lib.nix {};
};
}
// (vm_test "authelia" ./test/vm/authelia.nix)
// (vm_test "ldap" ./test/vm/ldap.nix)
// (vm_test "lib" ./test/vm/lib.nix)
// (vm_test "postgresql" ./test/vm/postgresql.nix)
// (vm_test "monitoring" ./test/vm/monitoring.nix)
// (vm_test "nextcloud" ./test/vm/nextcloud.nix)

View file

@ -1,13 +1,110 @@
{ lib }:
{
template = file: newPath: replacements:
{ pkgs, lib }:
rec {
replaceSecrets = { userConfig, resultPath, generator }:
let
templatePath = newPath + ".template";
configWithTemplates = withReplacements userConfig;
nonSecretConfigFile = pkgs.writeText "${resultPath}.template" (generator configWithTemplates);
replacements = getReplacements userConfig;
in
replaceSecretsScript {
file = nonSecretConfigFile;
inherit resultPath replacements;
};
template = file: newPath: replacements: replaceSecretsScript { inherit file replacements; resultPath = newPath; };
replaceSecretsScript = { file, resultPath, replacements }:
let
templatePath = resultPath + ".template";
sedPatterns = lib.strings.concatStringsSep " " (lib.attrsets.mapAttrsToList (from: to: "-e \"s|${from}|${to}|\"") replacements);
in
''
set -euo pipefail
set -x
ln -fs ${file} ${templatePath}
rm ${newPath} || :
sed ${sedPatterns} ${templatePath} > ${newPath}
rm -f ${resultPath}
${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath}
${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath} > ${resultPath}
'';
secretFileType = lib.types.submodule {
options = {
source = lib.mkOption {
type = lib.types.path;
description = "File containing the value.";
};
transform = lib.mkOption {
type = lib.types.raw;
description = "An optional function to transform the secret.";
default = null;
example = lib.literalExpression ''
v: "prefix-$${v}-suffix"
'';
};
};
};
secretName = name:
"%SECRET${lib.strings.toUpper (lib.strings.concatMapStrings (s: "_" + s) name)}%";
withReplacements = attrs:
let
valueOrReplacement = name: value:
if !(builtins.isAttrs value && value ? "source")
then value
else secretName name;
in
mapAttrsRecursiveCond (v: ! v ? "source") valueOrReplacement attrs;
getReplacements = attrs:
let
addNameField = name: value:
if !(builtins.isAttrs value && value ? "source")
then value
else value // { name = name; };
secretsWithName = mapAttrsRecursiveCond (v: ! v ? "source") addNameField attrs;
allSecrets = collect (v: builtins.isAttrs v && v ? "source") secretsWithName;
t = { transform ? null, ... }: if isNull transform then x: x else transform;
genReplacement = secret:
lib.attrsets.nameValuePair (secretName secret.name) ((t secret) "$(cat ${toString secret.source})");
in
lib.attrsets.listToAttrs (map genReplacement allSecrets);
# Inspired lib.attrsets.mapAttrsRecursiveCond but also recurses on lists.
mapAttrsRecursiveCond =
# A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set.
cond:
# A function, given a list of attribute names and a value, returns a new value.
f:
# Attribute set or list to recursively map over.
set:
let
recurse = path: val:
if builtins.isAttrs val && cond val
then lib.attrsets.mapAttrs (n: v: recurse (path ++ [n]) v) val
else if builtins.isList val && cond val
then lib.lists.imap0 (i: v: recurse (path ++ [(builtins.toString i)]) v) val
else f path val;
in recurse [] set;
# Like lib.attrsets.collect but also recurses on lists.
collect =
# Given an attribute's value, determine if recursion should stop.
pred:
# The attribute set to recursively collect.
attrs:
if pred attrs then
[ attrs ]
else if builtins.isAttrs attrs then
lib.lists.concatMap (collect pred) (lib.attrsets.attrValues attrs)
else if builtins.isList attrs then
lib.lists.concatMap (collect pred) attrs
else
[];
}

View file

@ -94,9 +94,54 @@ in
};
oidcClients = lib.mkOption {
type = lib.types.listOf lib.types.anything;
description = "OIDC clients";
default = [];
type = lib.types.listOf (lib.types.submodule {
freeformType = lib.types.attrsOf lib.types.anything;
options = {
id = lib.mkOption {
type = lib.types.str;
description = "Unique identifier of the OIDC client.";
};
description = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Human readable description of the OIDC client.";
default = null;
};
secret = lib.mkOption {
type = shblib.secretFileType;
description = "File containing the shared secret with the OIDC client.";
};
public = lib.mkOption {
type = lib.types.bool;
description = "If the OIDC client is public or not.";
default = false;
apply = v: if v then "true" else "false";
};
authorization_policy = lib.mkOption {
type = lib.types.enum [ "one_factor" "two_factor" ];
description = "Require one factor (password) or two factor (device) authentication.";
default = "one_factor";
};
redirect_uris = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "List of uris that are allowed to be redirected to.";
};
scopes = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Scopes to ask for";
example = [ "openid" "profile" "email" "groups" ];
default = [];
};
};
});
};
smtp = lib.mkOption {
@ -291,13 +336,13 @@ in
systemd.services."authelia-${fqdn}".preStart =
let
mkCfg = clients:
let
addTemplate = client: (builtins.removeAttrs client ["secretFile"]) // {secret = "%SECRET_${client.id}%";};
tmplFile = pkgs.writeText "oidc_clients.yaml" (lib.generators.toYAML {} {identity_providers.oidc.clients = map addTemplate clients;});
replace = client: {"%SECRET_${client.id}%" = "$(cat ${toString client.secretFile})";};
replacements = lib.foldl (container: client: container // (replace client) ) {} clients;
in
shblib.template tmplFile "/var/lib/authelia-${fqdn}/oidc_clients.yaml" replacements;
shblib.replaceSecrets {
userConfig = {
identity_providers.oidc.clients = clients;
};
resultPath = "/var/lib/authelia-${fqdn}/oidc_clients.yaml";
generator = lib.generators.toYAML {};
};
in
lib.mkBefore (mkCfg cfg.oidcClients);

View file

@ -4,6 +4,7 @@ let
cfg = config.shb.home-assistant;
contracts = pkgs.callPackage ../contracts {};
shblib = pkgs.callPackage ../../lib {};
fqdn = "${cfg.subdomain}.${cfg.domain}";
@ -18,6 +19,15 @@ let
export PATH=${pkgs.gnused}/bin:${pkgs.curl}/bin:${pkgs.jq}/bin
exec ${pkgs.bash}/bin/bash ${ldap_auth_script_repo}/example_configs/lldap-ha-auth.sh $@
'';
# Filter secrets from config. Secrets are those of the form { source = <path>; }
secrets = lib.attrsets.filterAttrs (k: v: builtins.isAttrs v) cfg.config;
nonSecrets = (lib.attrsets.filterAttrs (k: v: !(builtins.isAttrs v)) cfg.config);
configWithSecretsIncludes =
nonSecrets
// (lib.attrsets.mapAttrs (k: v: "!secret ${k}") secrets);
in
{
options.shb.home-assistant = {
@ -41,6 +51,41 @@ in
default = null;
};
config = lib.mkOption {
description = "See all available settings at https://www.home-assistant.io/docs/configuration/basic/";
type = lib.types.submodule {
freeformType = lib.types.attrsOf lib.types.str;
options = {
name = lib.mkOption {
type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
description = "Name of the Home Assistant instance.";
};
country = lib.mkOption {
type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
description = "Two letter country code where this instance is located.";
};
latitude = lib.mkOption {
type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
description = "Latitude where this instance is located.";
};
longitude = lib.mkOption {
type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
description = "Longitude where this instance is located.";
};
time_zone = lib.mkOption {
type = lib.types.oneOf [ lib.types.str shblib.secretFileType ];
description = "Timezone of this instance.";
example = "America/Los_Angeles";
};
unit_system = lib.mkOption {
type = lib.types.oneOf [ lib.types.str (lib.types.enum [ "metric" "us_customary" ]) ];
description = "Timezone of this instance.";
example = "America/Los_Angeles";
};
};
};
};
ldap = lib.mkOption {
description = ''
LDAP Integration App. [Manual](https://docs.nextcloud.com/server/latest/admin_manual/configuration_user/user_auth_ldap.html)
@ -91,12 +136,6 @@ in
};
};
sopsFile = lib.mkOption {
type = lib.types.path;
description = "Sops file location";
example = "secrets/homeassistant.yaml";
};
backupCfg = lib.mkOption {
type = lib.types.anything;
description = "Backup configuration for home-assistant";
@ -144,14 +183,8 @@ in
trusted_proxies = "127.0.0.1";
};
logger.default = "info";
homeassistant = {
homeassistant = configWithSecretsIncludes // {
external_url = "https://${cfg.subdomain}.${cfg.domain}";
name = "!secret name";
country = "!secret country";
latitude = "!secret latitude_home";
longitude = "!secret longitude_home";
time_zone = "!secret time_zone";
unit_system = "metric";
auth_providers =
(lib.optionals (!cfg.ldap.enable || cfg.ldap.keepDefaultAuth) [
{
@ -256,23 +289,18 @@ in
}
}
'';
storage = "${config.services.home-assistant.configDir}/.storage";
file = "${storage}/onboarding";
storage = "${config.services.home-assistant.configDir}";
file = "${storage}/.storage/onboarding";
in
''
if ! -f ${file}; then
mkdir -p ${storage} && cp ${onboarding} ${file}
fi
'');
sops.secrets."home-assistant" = {
inherit (cfg) sopsFile;
mode = "0440";
owner = "hass";
group = "hass";
path = "${config.services.home-assistant.configDir}/secrets.yaml";
restartUnits = [ "home-assistant.service" ];
};
'' + shblib.replaceSecrets {
userConfig = cfg.config;
resultPath = "${config.services.home-assistant.configDir}/secrets.yaml";
generator = lib.generators.toYAML {};
});
systemd.tmpfiles.rules = [
"f ${config.services.home-assistant.configDir}/automations.yaml 0755 hass hass"

View file

@ -348,19 +348,33 @@ in
</BrandingOptions>
'';
in
shblib.template ldapConfig "/var/lib/jellyfin/plugins/configurations/LDAP-Auth.xml" {
shblib.replaceSecretsScript {
file = ldapConfig;
resultPath = "/var/lib/jellyfin/plugins/configurations/LDAP-Auth.xml";
userConfig = {
"%LDAP_PASSWORD%" = "$(cat ${cfg.ldapPasswordFile})";
};
}
+ shblib.template ssoConfig "/var/lib/jellyfin/plugins/configurations/SSO-Auth.xml" {
+ shblib.replaceSecretsScript {
file = ssoConfig;
resultPath = "/var/lib/jellyfin/plugins/configurations/SSO-Auth.xml";
userConfig = {
"%SSO_SECRET%" = "$(cat ${cfg.ssoSecretFile})";
};
}
+ shblib.template brandingConfig "/var/lib/jellyfin/config/branding.xml" {"%a%" = "%a%";};
+ shblib.replaceSecretsScript {
file = brandingConfig;
resultPath = "/var/lib/jellyfin/config/branding.xml";
userConfig = {
"%a%" = "%a%";
};
};
shb.authelia.oidcClients = [
{
id = cfg.oidcClientID;
description = "Jellyfin";
secretFile = cfg.ssoSecretFile;
secret.source = cfg.ssoSecretFile;
public = false;
authorization_policy = "one_factor";
redirect_uris = [ "https://${cfg.subdomain}.${cfg.domain}/sso/OID/r/${cfg.oidcProvider}" ];

View file

@ -829,7 +829,7 @@ in
{
id = cfg.apps.sso.clientID;
description = "Nextcloud";
secretFile = cfg.apps.sso.secretFileForAuthelia;
secret.source = cfg.apps.sso.secretFileForAuthelia;
public = "false";
authorization_policy = cfg.apps.sso.authorization_policy;
redirect_uris = [ "${protocol}://${fqdnWithPort}/apps/oidc_login/oidc" ];

View file

@ -148,15 +148,14 @@ in
"f /var/lib/bitwarden_rs/vaultwarden.env 0640 vaultwarden vaultwarden"
];
systemd.services.vaultwarden.preStart =
let
envFile = pkgs.writeText "vaultwarden.env" ''
DATABASE_URL=postgresql://vaultwarden:%DB_PASSWORD%@127.0.0.1:5432/vaultwarden
SMTP_PASSWORD=%SMTP_PASSWORD%
'';
in
shblib.template envFile "/var/lib/bitwarden_rs/vaultwarden.env" {
"%DB_PASSWORD%" = "$(cat ${cfg.databasePasswordFile})";
"%SMTP_PASSWORD%" = "$(cat ${cfg.smtp.passwordFile})";
shblib.replaceSecrets {
userConfig = {
DATABASE_URL.source = cfg.databasePasswordFile;
DATABASE_URL.transform = v: "postgresql://vaultwarden:${v}@127.0.0.1:5432/vaultwarden";
SMTP_PASSWORD.source = cfg.smtp.passwordFile;
};
resultPath = "/var/lib/bitwarden_rs/vaultwarden.env";
generator = v: lib.generators.toINIWithGlobalSection {} { globalSection = v; };
};
shb.nginx.autheliaProtect = [

110
test/modules/lib.nix Normal file
View file

@ -0,0 +1,110 @@
{ pkgs, lib, ... }:
let
shblib = pkgs.callPackage ../../lib {};
in
{
# Tests that withReplacements can:
# - recurse in attrs and lists
# - .source field is understood
# - .transform field is understood
# - if .source field is found, ignores other fields
testLibWithReplacements = {
expected =
let
item = root: {
a = "A";
b = "%SECRET_${root}B%";
c = "%SECRET_${root}C%";
};
in
(item "") // {
nestedAttr = item "NESTEDATTR_";
nestedList = [ (item "NESTEDLIST_0_") ];
doubleNestedList = [ { n = (item "DOUBLENESTEDLIST_0_N_"); } ];
};
expr =
let
item = {
a = "A";
b.source = "/path/B";
b.transform = null;
c.source = "/path/C";
c.transform = v: "prefix-${v}-suffix";
c.other = "other";
};
in
shblib.withReplacements (
item // {
nestedAttr = item;
nestedList = [ item ];
doubleNestedList = [ { n = item; } ];
}
);
};
testLibWithReplacementsRootList = {
expected =
let
item = root: {
a = "A";
b = "%SECRET_${root}B%";
c = "%SECRET_${root}C%";
};
in
[
(item "0_")
(item "1_")
[ (item "2_0_") ]
[ { n = (item "3_0_N_"); } ]
];
expr =
let
item = {
a = "A";
b.source = "/path/B";
b.transform = null;
c.source = "/path/C";
c.transform = v: "prefix-${v}-suffix";
c.other = "other";
};
in
shblib.withReplacements [
item
item
[ item ]
[ { n = item; } ]
];
};
testLibGetReplacements = {
expected =
let
secrets = root: {
"%SECRET_${root}B%" = "$(cat /path/B)";
"%SECRET_${root}C%" = "prefix-$(cat /path/C)-suffix";
};
in
(secrets "") //
(secrets "NESTEDATTR_") //
(secrets "NESTEDLIST_0_") //
(secrets "DOUBLENESTEDLIST_0_N_");
expr =
let
item = {
a = "A";
b.source = "/path/B";
b.transform = null;
c.source = "/path/C";
c.transform = v: "prefix-${v}-suffix";
c.other = "other";
};
in
shblib.getReplacements (
item // {
nestedAttr = item;
nestedList = [ item ];
doubleNestedList = [ { n = item; } ];
}
);
};
}

View file

@ -10,7 +10,6 @@ in
imports = [
{
options = {
shb.ssl.enable = lib.mkEnableOption "ssl";
shb.backup = lib.mkOption { type = lib.types.anything; };
};
}
@ -49,7 +48,7 @@ in
{
id = "client1";
description = "My Client 1";
secretFile = pkgs.writeText "secret" "mysecuresecret";
secret.source = pkgs.writeText "secret" "mysecuresecret";
public = false;
authorization_policy = "one_factor";
redirect_uris = [ "http://client1.machine/redirect" ];
@ -57,7 +56,7 @@ in
{
id = "client2";
description = "My Client 2";
secretFile = pkgs.writeText "secret" "myothersecret";
secret.source = pkgs.writeText "secret" "myothersecret";
public = false;
authorization_policy = "one_factor";
redirect_uris = [ "http://client2.machine/redirect" ];

82
test/vm/lib.nix Normal file
View file

@ -0,0 +1,82 @@
{ pkgs, lib, ... }:
let
shblib = pkgs.callPackage ../../lib {};
in
{
template =
let
aSecret = pkgs.writeText "a-secret.txt" "Secret of A";
bSecret = pkgs.writeText "b-secret.txt" "Secret of B";
userConfig = {
a.a.source = aSecret;
b.source = bSecret;
b.transform = v: "prefix-${v}-suffix";
c = "not secret C";
d.d = "not secret D";
};
wantedConfig = {
a.a = "Secret of A";
b = "prefix-Secret of B-suffix";
c = "not secret C";
d.d = "not secret D";
};
configWithTemplates = shblib.withReplacements userConfig;
nonSecretConfigFile = pkgs.writeText "config.yaml.template" (lib.generators.toJSON {} configWithTemplates);
replacements = shblib.getReplacements userConfig;
replaceInTemplate = shblib.replaceSecretsScript {
file = nonSecretConfigFile;
resultPath = "/var/lib/config.yaml";
inherit replacements;
};
replaceInTemplate2 = shblib.replaceSecrets {
inherit userConfig;
resultPath = "/var/lib/config2.yaml";
generator = lib.generators.toJSON {};
};
in
pkgs.nixosTest {
name = "lib-template";
nodes.machine = { config, pkgs, ... }:
{
imports = [
{
options = {
libtest.config = lib.mkOption {
type = lib.types.attrsOf (lib.types.oneOf [ lib.types.str shblib.secretFileType ]);
};
};
}
];
system.activationScripts = {
libtest = replaceInTemplate;
libtest2 = replaceInTemplate2;
};
};
testScript = { nodes, ... }: ''
import json
start_all()
wantedConfig = json.loads('${lib.generators.toJSON {} wantedConfig}')
gotConfig = json.loads(machine.succeed("cat /var/lib/config.yaml"))
gotConfig2 = json.loads(machine.succeed("cat /var/lib/config2.yaml"))
# For debugging purpose
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate2" replaceInTemplate2}"))
if wantedConfig != gotConfig:
raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
if wantedConfig != gotConfig2:
raise Exception("\nwantedConfig: {}\n!= gotConfig2: {}".format(wantedConfig, gotConfig))
'';
};
}