Fix backup contract secrets (#280)
This commit is contained in:
parent
0fa4a42be7
commit
10dea06ec1
7 changed files with 145 additions and 22 deletions
|
@ -1,4 +1,8 @@
|
||||||
{ pkgs, lib }:
|
{ pkgs, lib }:
|
||||||
|
let
|
||||||
|
inherit (builtins) isAttrs hasAttr;
|
||||||
|
inherit (lib) concatStringsSep;
|
||||||
|
in
|
||||||
rec {
|
rec {
|
||||||
# Replace secrets in a file.
|
# Replace secrets in a file.
|
||||||
# - userConfig is an attrset that will produce a config file.
|
# - userConfig is an attrset that will produce a config file.
|
||||||
|
@ -228,8 +232,47 @@ rec {
|
||||||
results = pkgs.lib.runTests tests;
|
results = pkgs.lib.runTests tests;
|
||||||
in
|
in
|
||||||
if results != [ ] then
|
if results != [ ] then
|
||||||
builtins.throw (builtins.concatStringsSep "\n" (map resultToString results))
|
builtins.throw (builtins.concatStringsSep "\n" (map resultToString (lib.traceValSeqN 3 results)))
|
||||||
else
|
else
|
||||||
pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
|
pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
|
||||||
|
|
||||||
|
|
||||||
|
genConfigOutOfBandSystemd = { config, configLocation, generator }:
|
||||||
|
{
|
||||||
|
loadCredentials = getLoadCredentials "source" config;
|
||||||
|
preStart = lib.mkBefore (replaceSecrets {
|
||||||
|
userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
|
||||||
|
resultPath = configLocation;
|
||||||
|
inherit generator;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
updateToLoadCredentials = sourceField: rootDir: attrs:
|
||||||
|
let
|
||||||
|
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
|
||||||
|
|
||||||
|
valueOrLoadCredential = path: value:
|
||||||
|
if ! (hasPlaceholderField value)
|
||||||
|
then value
|
||||||
|
else value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
|
||||||
|
in
|
||||||
|
mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) valueOrLoadCredential attrs;
|
||||||
|
|
||||||
|
getLoadCredentials = sourceField: attrs:
|
||||||
|
let
|
||||||
|
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
|
||||||
|
|
||||||
|
addPathField = path: value:
|
||||||
|
if ! (hasPlaceholderField value)
|
||||||
|
then value
|
||||||
|
else value // { inherit path; };
|
||||||
|
|
||||||
|
secretsWithPath = mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) addPathField attrs;
|
||||||
|
|
||||||
|
allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
|
||||||
|
|
||||||
|
genLoadCredentials = secret:
|
||||||
|
"${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
|
||||||
|
in
|
||||||
|
map genLoadCredentials allSecrets;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
{ config, options, pkgs, lib, ... }:
|
{ config, pkgs, lib, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
cfg = config.shb.monitoring;
|
cfg = config.shb.monitoring;
|
||||||
|
|
|
@ -13,6 +13,26 @@ let
|
||||||
type = lib.types.path;
|
type = lib.types.path;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
user = lib.mkOption {
|
||||||
|
description = ''
|
||||||
|
Unix user doing the backups.
|
||||||
|
|
||||||
|
For Restic, the same user must be used for all instances.
|
||||||
|
'';
|
||||||
|
type = lib.types.str;
|
||||||
|
default = cfg.user;
|
||||||
|
};
|
||||||
|
|
||||||
|
group = lib.mkOption {
|
||||||
|
description = ''
|
||||||
|
Unix group doing the backups.
|
||||||
|
|
||||||
|
For Restic, the same group must be used for all instances.
|
||||||
|
'';
|
||||||
|
type = lib.types.str;
|
||||||
|
default = cfg.group;
|
||||||
|
};
|
||||||
|
|
||||||
sourceDirectories = lib.mkOption {
|
sourceDirectories = lib.mkOption {
|
||||||
description = "Source directories.";
|
description = "Source directories.";
|
||||||
type = lib.types.nonEmptyListOf lib.types.str;
|
type = lib.types.nonEmptyListOf lib.types.str;
|
||||||
|
@ -33,13 +53,15 @@ let
|
||||||
description = "Repository location";
|
description = "Repository location";
|
||||||
};
|
};
|
||||||
|
|
||||||
extraSecrets = lib.mkOption {
|
secrets = lib.mkOption {
|
||||||
type = lib.types.attrsOf shblib.secretFileType;
|
type = lib.types.attrsOf shblib.secretFileType;
|
||||||
default = {};
|
default = {};
|
||||||
description = ''
|
description = ''
|
||||||
Extra secrets needed to access the repository where the backups will be stored.
|
Secrets needed to access the repository where the backups will be stored.
|
||||||
|
|
||||||
|
See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
|
||||||
|
and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
|
||||||
|
|
||||||
See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example.
|
|
||||||
'';
|
'';
|
||||||
example = lib.literalExpression ''
|
example = lib.literalExpression ''
|
||||||
{
|
{
|
||||||
|
@ -154,6 +176,17 @@ in
|
||||||
enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
|
enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
|
||||||
in lib.mkMerge [
|
in lib.mkMerge [
|
||||||
{
|
{
|
||||||
|
assertions = [
|
||||||
|
{
|
||||||
|
assertion = lib.all (x: x.user == cfg.user) (lib.mapAttrsToList (n: v: v)cfg.instances);
|
||||||
|
message = "All Restic instances must have the same user as 'shb.restic.user'.";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
assertion = lib.all (x: x.group == cfg.group) (lib.mapAttrsToList (n: v: v) cfg.instances);
|
||||||
|
message = "All Restic instances must have the same group as 'shb.restic.group'.";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
users.users = {
|
users.users = {
|
||||||
${cfg.user} = {
|
${cfg.user} = {
|
||||||
name = cfg.user;
|
name = cfg.user;
|
||||||
|
@ -195,12 +228,6 @@ in
|
||||||
backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
|
backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
|
||||||
|
|
||||||
backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
|
backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
|
||||||
} // lib.attrsets.optionalAttrs (repository.extraSecrets != {}) {
|
|
||||||
environmentFile = shblib.replaceSecrets {
|
|
||||||
userConfig = repository.extraSecrets;
|
|
||||||
resultPath = "/var/lib/backup/${name}";
|
|
||||||
generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} v);
|
|
||||||
};
|
|
||||||
} // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
|
} // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
|
||||||
exclude = instance.excludePatterns;
|
exclude = instance.excludePatterns;
|
||||||
};
|
};
|
||||||
|
@ -212,12 +239,42 @@ in
|
||||||
|
|
||||||
systemd.services =
|
systemd.services =
|
||||||
let
|
let
|
||||||
mkRepositorySettings = name: instance: repository: {
|
mkRepositorySettings = name: instance: repository:
|
||||||
"restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = {
|
let
|
||||||
Nice = cfg.performance.niceness;
|
serviceName = "restic-backups-${name}_${repoSlugName repository.path}";
|
||||||
IOSchedulingClass = cfg.performance.ioSchedulingClass;
|
in
|
||||||
IOSchedulingPriority = cfg.performance.ioPriority;
|
{
|
||||||
};
|
${serviceName} = lib.mkMerge [
|
||||||
|
{
|
||||||
|
serviceConfig = {
|
||||||
|
Nice = cfg.performance.niceness;
|
||||||
|
IOSchedulingClass = cfg.performance.ioSchedulingClass;
|
||||||
|
IOSchedulingPriority = cfg.performance.ioPriority;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
(lib.attrsets.optionalAttrs (repository.secrets != {})
|
||||||
|
{
|
||||||
|
serviceConfig.EnvironmentFile = [
|
||||||
|
"/run/secrets/restic/${serviceName}"
|
||||||
|
];
|
||||||
|
after = [ "${serviceName}-pre.service" ];
|
||||||
|
requires = [ "${serviceName}-pre.service" ];
|
||||||
|
})
|
||||||
|
];
|
||||||
|
|
||||||
|
"${serviceName}-pre" = lib.mkIf (repository.secrets != {})
|
||||||
|
(let
|
||||||
|
script = shblib.genConfigOutOfBandSystemd {
|
||||||
|
config = repository.secrets;
|
||||||
|
configLocation = "/run/secrets/restic/${serviceName}";
|
||||||
|
generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} { globalSection = v; });
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
script = script.preStart;
|
||||||
|
serviceConfig.Type = "oneshot";
|
||||||
|
serviceConfig.LoadCredential = script.loadCredentials;
|
||||||
|
});
|
||||||
};
|
};
|
||||||
mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
|
mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
|
||||||
in
|
in
|
||||||
|
|
|
@ -76,8 +76,8 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
|
||||||
};
|
};
|
||||||
|
|
||||||
+ extraSecrets = {
|
+ extraSecrets = {
|
||||||
+ AWS_ACCESS_KEY_ID="<path/to/access_key_id>";
|
+ AWS_ACCESS_KEY_ID.source="<path/to/access_key_id>";
|
||||||
+ AWS_SECRET_ACCESS_KEY="<path/to/secret_access_key>";
|
+ AWS_SECRET_ACCESS_KEY.source="<path/to/secret_access_key>";
|
||||||
+ };
|
+ };
|
||||||
}];
|
}];
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
{ config, options, pkgs, lib, ... }:
|
{ config, pkgs, lib, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
cfg = config.shb.arr;
|
cfg = config.shb.arr;
|
||||||
|
|
|
@ -1,8 +1,7 @@
|
||||||
{ config, options, pkgs, lib, ... }:
|
{ config, pkgs, lib, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
cfg = config.shb.nextcloud;
|
cfg = config.shb.nextcloud;
|
||||||
opt = options.shb.nextcloud;
|
|
||||||
|
|
||||||
fqdn = "${cfg.subdomain}.${cfg.domain}";
|
fqdn = "${cfg.subdomain}.${cfg.domain}";
|
||||||
fqdnWithPort = if isNull cfg.port then fqdn else "${fqdn}:${toString cfg.port}";
|
fqdnWithPort = if isNull cfg.port then fqdn else "${fqdn}:${toString cfg.port}";
|
||||||
|
|
|
@ -3,6 +3,7 @@ let
|
||||||
pkgs' = pkgs;
|
pkgs' = pkgs;
|
||||||
|
|
||||||
testLib = pkgs.callPackage ../common.nix {};
|
testLib = pkgs.callPackage ../common.nix {};
|
||||||
|
shblib = pkgs.callPackage ../../lib {};
|
||||||
|
|
||||||
base = testLib.base [
|
base = testLib.base [
|
||||||
../../modules/blocks/restic.nix
|
../../modules/blocks/restic.nix
|
||||||
|
@ -38,6 +39,12 @@ in
|
||||||
OnCalendar = "00:00:00";
|
OnCalendar = "00:00:00";
|
||||||
RandomizedDelaySec = "5h";
|
RandomizedDelaySec = "5h";
|
||||||
};
|
};
|
||||||
|
# Those are not needed by the repository but are still included
|
||||||
|
# so we can test them in the hooks section.
|
||||||
|
secrets = {
|
||||||
|
A.source = pkgs.writeText "A" "secretA";
|
||||||
|
B.source = pkgs.writeText "B" "secretB";
|
||||||
|
};
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
path = "/opt/repos/B";
|
path = "/opt/repos/B";
|
||||||
|
@ -47,6 +54,23 @@ in
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
|
hooks.before_backup = [''
|
||||||
|
echo $RUNTIME_DIRECTORY
|
||||||
|
if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
|
||||||
|
if ! [ -f /run/secrets/restic/restic-backups-testinstance_opt_repos_A ]; then
|
||||||
|
exit 10
|
||||||
|
fi
|
||||||
|
if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
|
||||||
|
echo "A:$A"
|
||||||
|
exit 11
|
||||||
|
fi
|
||||||
|
if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
|
||||||
|
echo "A:$A"
|
||||||
|
exit 12
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
''];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue