hosting/selfhostblocks
1
0
Fork 0
Code Issues Activity

Fix backup contract secrets (#280)

Browse source
This commit is contained in:
Pierre Penninckx 2024-08-22 12:48:36 -07:00 committed by GitHub
parent 0fa4a42be7
commit 10dea06ec1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 145 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
lib/default.nix
View file

@ -1,4 +1,8 @@
{ pkgs, lib }:
let
inherit (builtins) isAttrs hasAttr;
inherit (lib) concatStringsSep;
in
rec {
# Replace secrets in a file.
# - userConfig is an attrset that will produce a config file.
@ -228,8 +232,47 @@ rec {
results = pkgs.lib.runTests tests;
in
if results != [ ] then
builtins.throw (builtins.concatStringsSep "\n" (map resultToString results))
builtins.throw (builtins.concatStringsSep "\n" (map resultToString (lib.traceValSeqN 3 results)))
else
pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
genConfigOutOfBandSystemd = { config, configLocation, generator }:
{
loadCredentials = getLoadCredentials "source" config;
preStart = lib.mkBefore (replaceSecrets {
userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
resultPath = configLocation;
inherit generator;
});
};
updateToLoadCredentials = sourceField: rootDir: attrs:
let
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
valueOrLoadCredential = path: value:
if ! (hasPlaceholderField value)
then value
else value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
in
mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) valueOrLoadCredential attrs;
getLoadCredentials = sourceField: attrs:
let
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
addPathField = path: value:
if ! (hasPlaceholderField value)
then value
else value // { inherit path; };
secretsWithPath = mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) addPathField attrs;
allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
genLoadCredentials = secret:
"${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
in
map genLoadCredentials allSecrets;
}

2
modules/blocks/monitoring.nix
View file

@ -1,4 +1,4 @@
{ config, options, pkgs, lib, ... }:
{ config, pkgs, lib, ... }:
let
cfg = config.shb.monitoring;

87
modules/blocks/restic.nix
View file

@ -13,6 +13,26 @@ let
type = lib.types.path;
};
user = lib.mkOption {
description = ''
Unix user doing the backups.
For Restic, the same user must be used for all instances.
'';
type = lib.types.str;
default = cfg.user;
};
group = lib.mkOption {
description = ''
Unix group doing the backups.
For Restic, the same group must be used for all instances.
'';
type = lib.types.str;
default = cfg.group;
};
sourceDirectories = lib.mkOption {
description = "Source directories.";
type = lib.types.nonEmptyListOf lib.types.str;
@ -33,13 +53,15 @@ let
description = "Repository location";
};
extraSecrets = lib.mkOption {
secrets = lib.mkOption {
type = lib.types.attrsOf shblib.secretFileType;
default = {};
description = ''
Extra secrets needed to access the repository where the backups will be stored.
Secrets needed to access the repository where the backups will be stored.
See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example.
'';
example = lib.literalExpression ''
{
@ -154,6 +176,17 @@ in
enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
in lib.mkMerge [
{
assertions = [
{
assertion = lib.all (x: x.user == cfg.user) (lib.mapAttrsToList (n: v: v)cfg.instances);
message = "All Restic instances must have the same user as 'shb.restic.user'.";
}
{
assertion = lib.all (x: x.group == cfg.group) (lib.mapAttrsToList (n: v: v) cfg.instances);
message = "All Restic instances must have the same group as 'shb.restic.group'.";
}
];
users.users = {
${cfg.user} = {
name = cfg.user;
@ -195,12 +228,6 @@ in
backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
} // lib.attrsets.optionalAttrs (repository.extraSecrets != {}) {
environmentFile = shblib.replaceSecrets {
userConfig = repository.extraSecrets;
resultPath = "/var/lib/backup/${name}";
generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} v);
};
} // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
exclude = instance.excludePatterns;
};
@ -212,12 +239,42 @@ in
systemd.services =
let
mkRepositorySettings = name: instance: repository: {
"restic-backups-${name}_${repoSlugName repository.path}".serviceConfig = {
Nice = cfg.performance.niceness;
IOSchedulingClass = cfg.performance.ioSchedulingClass;
IOSchedulingPriority = cfg.performance.ioPriority;
};
mkRepositorySettings = name: instance: repository:
let
serviceName = "restic-backups-${name}_${repoSlugName repository.path}";
in
{
${serviceName} = lib.mkMerge [
{
serviceConfig = {
Nice = cfg.performance.niceness;
IOSchedulingClass = cfg.performance.ioSchedulingClass;
IOSchedulingPriority = cfg.performance.ioPriority;
};
}
(lib.attrsets.optionalAttrs (repository.secrets != {})
{
serviceConfig.EnvironmentFile = [
"/run/secrets/restic/${serviceName}"
];
after = [ "${serviceName}-pre.service" ];
requires = [ "${serviceName}-pre.service" ];
})
];
"${serviceName}-pre" = lib.mkIf (repository.secrets != {})
(let
script = shblib.genConfigOutOfBandSystemd {
config = repository.secrets;
configLocation = "/run/secrets/restic/${serviceName}";
generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} { globalSection = v; });
};
in
{
script = script.preStart;
serviceConfig.Type = "oneshot";
serviceConfig.LoadCredential = script.loadCredentials;
});
};
mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
in

4
modules/blocks/restic/docs/default.md
View file

@ -76,8 +76,8 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
};
+ extraSecrets = {
+ AWS_ACCESS_KEY_ID="<path/to/access_key_id>";
+ AWS_SECRET_ACCESS_KEY="<path/to/secret_access_key>";
+ AWS_ACCESS_KEY_ID.source="<path/to/access_key_id>";
+ AWS_SECRET_ACCESS_KEY.source="<path/to/secret_access_key>";
+ };
}];
}

2
modules/services/arr.nix
View file

@ -1,4 +1,4 @@
{ config, options, pkgs, lib, ... }:
{ config, pkgs, lib, ... }:
let
cfg = config.shb.arr;

3
modules/services/nextcloud-server.nix
View file

@ -1,8 +1,7 @@
{ config, options, pkgs, lib, ... }:
{ config, pkgs, lib, ... }:
let
cfg = config.shb.nextcloud;
opt = options.shb.nextcloud;
fqdn = "${cfg.subdomain}.${cfg.domain}";
fqdnWithPort = if isNull cfg.port then fqdn else "${fqdn}:${toString cfg.port}";

24
test/blocks/restic.nix
View file

@ -3,6 +3,7 @@ let
pkgs' = pkgs;
testLib = pkgs.callPackage ../common.nix {};
shblib = pkgs.callPackage ../../lib {};
base = testLib.base [
../../modules/blocks/restic.nix
@ -38,6 +39,12 @@ in
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
# Those are not needed by the repository but are still included
# so we can test them in the hooks section.
secrets = {
A.source = pkgs.writeText "A" "secretA";
B.source = pkgs.writeText "B" "secretB";
};
}
{
path = "/opt/repos/B";
@ -47,6 +54,23 @@ in
};
}
];
hooks.before_backup = [''
echo $RUNTIME_DIRECTORY
if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
if ! [ -f /run/secrets/restic/restic-backups-testinstance_opt_repos_A ]; then
exit 10
fi
if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
echo "A:$A"
exit 11
fi
if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
echo "A:$A"
exit 12
fi
fi
''];
};
};