1
0
Fork 0

Fix backup contract secrets (#280)

This commit is contained in:
Pierre Penninckx 2024-08-22 12:48:36 -07:00 committed by GitHub
parent 0fa4a42be7
commit 10dea06ec1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 145 additions and 22 deletions

View file

@ -1,4 +1,8 @@
{ pkgs, lib }: { pkgs, lib }:
let
inherit (builtins) isAttrs hasAttr;
inherit (lib) concatStringsSep;
in
rec { rec {
# Replace secrets in a file. # Replace secrets in a file.
# - userConfig is an attrset that will produce a config file. # - userConfig is an attrset that will produce a config file.
@ -228,8 +232,47 @@ rec {
results = pkgs.lib.runTests tests; results = pkgs.lib.runTests tests;
in in
if results != [ ] then if results != [ ] then
builtins.throw (builtins.concatStringsSep "\n" (map resultToString results)) builtins.throw (builtins.concatStringsSep "\n" (map resultToString (lib.traceValSeqN 3 results)))
else else
pkgs.runCommand "nix-flake-tests-success" { } "echo > $out"; pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
genConfigOutOfBandSystemd = { config, configLocation, generator }:
{
loadCredentials = getLoadCredentials "source" config;
preStart = lib.mkBefore (replaceSecrets {
userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
resultPath = configLocation;
inherit generator;
});
};
updateToLoadCredentials = sourceField: rootDir: attrs:
let
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
valueOrLoadCredential = path: value:
if ! (hasPlaceholderField value)
then value
else value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
in
mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) valueOrLoadCredential attrs;
getLoadCredentials = sourceField: attrs:
let
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
addPathField = path: value:
if ! (hasPlaceholderField value)
then value
else value // { inherit path; };
secretsWithPath = mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) addPathField attrs;
allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
genLoadCredentials = secret:
"${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
in
map genLoadCredentials allSecrets;
} }

View file

@ -1,4 +1,4 @@
{ config, options, pkgs, lib, ... }: { config, pkgs, lib, ... }:
let let
cfg = config.shb.monitoring; cfg = config.shb.monitoring;

View file

@ -13,6 +13,26 @@ let
type = lib.types.path; type = lib.types.path;
}; };
user = lib.mkOption {
description = ''
Unix user doing the backups.
For Restic, the same user must be used for all instances.
'';
type = lib.types.str;
default = cfg.user;
};
group = lib.mkOption {
description = ''
Unix group doing the backups.
For Restic, the same group must be used for all instances.
'';
type = lib.types.str;
default = cfg.group;
};
sourceDirectories = lib.mkOption { sourceDirectories = lib.mkOption {
description = "Source directories."; description = "Source directories.";
type = lib.types.nonEmptyListOf lib.types.str; type = lib.types.nonEmptyListOf lib.types.str;
@ -33,13 +53,15 @@ let
description = "Repository location"; description = "Repository location";
}; };
extraSecrets = lib.mkOption { secrets = lib.mkOption {
type = lib.types.attrsOf shblib.secretFileType; type = lib.types.attrsOf shblib.secretFileType;
default = {}; default = {};
description = '' description = ''
Extra secrets needed to access the repository where the backups will be stored. Secrets needed to access the repository where the backups will be stored.
See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example.
''; '';
example = lib.literalExpression '' example = lib.literalExpression ''
{ {
@ -154,6 +176,17 @@ in
enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances; enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
in lib.mkMerge [ in lib.mkMerge [
{ {
assertions = [
{
assertion = lib.all (x: x.user == cfg.user) (lib.mapAttrsToList (n: v: v)cfg.instances);
message = "All Restic instances must have the same user as 'shb.restic.user'.";
}
{
assertion = lib.all (x: x.group == cfg.group) (lib.mapAttrsToList (n: v: v) cfg.instances);
message = "All Restic instances must have the same group as 'shb.restic.group'.";
}
];
users.users = { users.users = {
${cfg.user} = { ${cfg.user} = {
name = cfg.user; name = cfg.user;
@ -195,12 +228,6 @@ in
backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup; backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup; backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
} // lib.attrsets.optionalAttrs (repository.extraSecrets != {}) {
environmentFile = shblib.replaceSecrets {
userConfig = repository.extraSecrets;
resultPath = "/var/lib/backup/${name}";
generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} v);
};
} // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) { } // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
exclude = instance.excludePatterns; exclude = instance.excludePatterns;
}; };
@ -212,12 +239,42 @@ in
systemd.services = systemd.services =
let let
mkRepositorySettings = name: instance: repository: { mkRepositorySettings = name: instance: repository:
"restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = { let
Nice = cfg.performance.niceness; serviceName = "restic-backups-${name}_${repoSlugName repository.path}";
IOSchedulingClass = cfg.performance.ioSchedulingClass; in
IOSchedulingPriority = cfg.performance.ioPriority; {
}; ${serviceName} = lib.mkMerge [
{
serviceConfig = {
Nice = cfg.performance.niceness;
IOSchedulingClass = cfg.performance.ioSchedulingClass;
IOSchedulingPriority = cfg.performance.ioPriority;
};
}
(lib.attrsets.optionalAttrs (repository.secrets != {})
{
serviceConfig.EnvironmentFile = [
"/run/secrets/restic/${serviceName}"
];
after = [ "${serviceName}-pre.service" ];
requires = [ "${serviceName}-pre.service" ];
})
];
"${serviceName}-pre" = lib.mkIf (repository.secrets != {})
(let
script = shblib.genConfigOutOfBandSystemd {
config = repository.secrets;
configLocation = "/run/secrets/restic/${serviceName}";
generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} { globalSection = v; });
};
in
{
script = script.preStart;
serviceConfig.Type = "oneshot";
serviceConfig.LoadCredential = script.loadCredentials;
});
}; };
mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
in in

View file

@ -76,8 +76,8 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
}; };
+ extraSecrets = { + extraSecrets = {
+ AWS_ACCESS_KEY_ID="<path/to/access_key_id>"; + AWS_ACCESS_KEY_ID.source="<path/to/access_key_id>";
+ AWS_SECRET_ACCESS_KEY="<path/to/secret_access_key>"; + AWS_SECRET_ACCESS_KEY.source="<path/to/secret_access_key>";
+ }; + };
}]; }];
} }

View file

@ -1,4 +1,4 @@
{ config, options, pkgs, lib, ... }: { config, pkgs, lib, ... }:
let let
cfg = config.shb.arr; cfg = config.shb.arr;

View file

@ -1,8 +1,7 @@
{ config, options, pkgs, lib, ... }: { config, pkgs, lib, ... }:
let let
cfg = config.shb.nextcloud; cfg = config.shb.nextcloud;
opt = options.shb.nextcloud;
fqdn = "${cfg.subdomain}.${cfg.domain}"; fqdn = "${cfg.subdomain}.${cfg.domain}";
fqdnWithPort = if isNull cfg.port then fqdn else "${fqdn}:${toString cfg.port}"; fqdnWithPort = if isNull cfg.port then fqdn else "${fqdn}:${toString cfg.port}";

View file

@ -3,6 +3,7 @@ let
pkgs' = pkgs; pkgs' = pkgs;
testLib = pkgs.callPackage ../common.nix {}; testLib = pkgs.callPackage ../common.nix {};
shblib = pkgs.callPackage ../../lib {};
base = testLib.base [ base = testLib.base [
../../modules/blocks/restic.nix ../../modules/blocks/restic.nix
@ -38,6 +39,12 @@ in
OnCalendar = "00:00:00"; OnCalendar = "00:00:00";
RandomizedDelaySec = "5h"; RandomizedDelaySec = "5h";
}; };
# Those are not needed by the repository but are still included
# so we can test them in the hooks section.
secrets = {
A.source = pkgs.writeText "A" "secretA";
B.source = pkgs.writeText "B" "secretB";
};
} }
{ {
path = "/opt/repos/B"; path = "/opt/repos/B";
@ -47,6 +54,23 @@ in
}; };
} }
]; ];
hooks.before_backup = [''
echo $RUNTIME_DIRECTORY
if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
if ! [ -f /run/secrets/restic/restic-backups-testinstance_opt_repos_A ]; then
exit 10
fi
if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
echo "A:$A"
exit 11
fi
if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
echo "A:$A"
exit 12
fi
fi
''];
}; };
}; };