hosting/selfhostblocks
1
0
Fork 0
Code Issues Activity

simplify backup of services by using user

Browse source
This commit is contained in:
ibizaman 2024-08-23 22:47:50 +02:00
parent 2e8b3fb166
commit dd4a13ad43
13 changed files with 178 additions and 364 deletions

6
modules/blocks/ldap.nix
View file

@ -106,6 +106,7 @@ in
'';
readOnly = true;
default = {
user = "lldap";
sourceDirectories = [
"/var/lib/lldap"
];
@ -139,10 +140,7 @@ in
group = "lldap";
isSystemUser = true;
};
users.groups.lldap = {
members = [ "backup" ];
};
users.groups.lldap = {};
services.lldap = {
enable = true;

48
modules/blocks/restic.nix
View file

@ -15,12 +15,9 @@ let
user = lib.mkOption {
description = ''
Unix user doing the backups.
For Restic, the same user must be used for all instances.
Unix user doing the backups. Must be the user owning the files to be backed up.
'';
type = lib.types.str;
default = cfg.user;
};
sourceDirectories = lib.mkOption {
@ -115,12 +112,6 @@ let
in
{
options.shb.restic = {
user = lib.mkOption {
description = "Unix user doing the backups.";
type = lib.types.str;
default = "backup";
};
instances = lib.mkOption {
description = "Each instance is a backup setting";
default = {};
@ -159,34 +150,6 @@ in
let
enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
in lib.mkMerge [
{
assertions = [
{
assertion = lib.all (x: x.user == cfg.user) (lib.mapAttrsToList (n: v: v)cfg.instances);
message = "All Restic instances must have the same user as 'shb.restic.user'.";
}
{
assertion = lib.all (x: x.group == cfg.group) (lib.mapAttrsToList (n: v: v) cfg.instances);
message = "All Restic instances must have the same group as 'shb.restic.group'.";
}
];
users.users = {
${cfg.user} = {
name = cfg.user;
group = cfg.group;
home = lib.mkForce "/var/lib/${cfg.user}";
createHome = true;
isSystemUser = true;
extraGroups = [ "keys" ];
};
};
users.groups = {
${cfg.group} = {
name = cfg.group;
};
};
}
{
environment.systemPackages = lib.optionals (enabledInstances != {}) [ pkgs.restic ];
@ -204,7 +167,8 @@ in
let
mkRepositorySettings = name: instance: repository: {
"${name}_${repoSlugName repository.path}" = {
inherit (cfg) user;
inherit (instance) user;
repository = repository.path;
paths = instance.sourceDirectories;
@ -244,12 +208,13 @@ in
Nice = cfg.performance.niceness;
IOSchedulingClass = cfg.performance.ioSchedulingClass;
IOSchedulingPriority = cfg.performance.ioPriority;
BindReadOnlyPaths = instance.sourceDirectories;
};
}
(lib.attrsets.optionalAttrs (repository.secrets != {})
{
serviceConfig.EnvironmentFile = [
"/run/secrets/restic/${serviceName}"
"/run/secrets_restic/${serviceName}"
];
after = [ "${serviceName}-pre.service" ];
requires = [ "${serviceName}-pre.service" ];
@ -260,8 +225,9 @@ in
(let
script = shblib.genConfigOutOfBandSystemd {
config = repository.secrets;
configLocation = "/run/secrets/restic/${serviceName}";
configLocation = "/run/secrets_restic/${serviceName}";
generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} { globalSection = v; });
user = instance.user;
};
in
{

22
modules/services/arr.nix
View file

@ -277,20 +277,6 @@ let
];
};
backup = name: {
systemd.tmpfiles.rules = [
"d '${config.shb.arr.${name}.dataDir}' 0750 ${config.services.${name}.user} ${config.services.${name}.group} - -"
];
users.groups.${name} = {
members = [ "backup" ];
};
systemd.services.${name}.serviceConfig = {
# Setup permissions needed for backups, as the backup user is member of the jellyfin group.
UMask = lib.mkForce "0027";
StateDirectoryMode = lib.mkForce "0750";
};
};
appOption = name: c: lib.nameValuePair name (lib.mkOption {
description = "Configuration for ${name}";
default = {};
@ -347,6 +333,7 @@ let
'';
readOnly = true;
default = {
user = name;
sourceDirectories = [
cfg.${name}.dataDir
];
@ -386,7 +373,6 @@ in
shb.nginx.vhosts = [ (vhosts {} cfg') ];
}))
(lib.mkIf cfg.radarr.enable (backup "radarr"))
(lib.mkIf cfg.sonarr.enable (
let
@ -416,7 +402,6 @@ in
shb.nginx.vhosts = [ (vhosts {} cfg') ];
}))
(lib.mkIf cfg.sonarr.enable (backup "sonarr"))
(lib.mkIf cfg.bazarr.enable (
let
@ -443,7 +428,6 @@ in
shb.nginx.vhosts = [ (vhosts {} cfg') ];
}))
(lib.mkIf cfg.bazarr.enable (backup "bazarr"))
(lib.mkIf cfg.readarr.enable (
let
@ -465,7 +449,6 @@ in
shb.nginx.vhosts = [ (vhosts {} cfg') ];
}))
(lib.mkIf cfg.readarr.enable (backup "readarr"))
(lib.mkIf cfg.lidarr.enable (
let
@ -492,7 +475,6 @@ in
shb.nginx.vhosts = [ (vhosts {} cfg') ];
}))
(lib.mkIf cfg.lidarr.enable (backup "lidarr"))
(lib.mkIf cfg.jackett.enable (
let
@ -503,6 +485,7 @@ in
enable = true;
dataDir = "/var/lib/jackett";
};
# TODO: avoid implicitly relying on the media group
users.users.jackett = {
extraGroups = [ "media" ];
};
@ -516,6 +499,5 @@ in
extraBypassResources = [ "^/dl.*" ];
} cfg') ];
}))
(lib.mkIf cfg.jackett.enable (backup "jackett"))
];
}

12
modules/services/audiobookshelf.nix
View file

@ -100,6 +100,7 @@ in
'';
readOnly = true;
default = {
user = "audiobookshelf";
sourceDirectories = [
"/var/lib/audiobookshelf"
];
@ -162,17 +163,6 @@ in
];
}
];
# We want audiobookshelf to create files in the media group and to make those files group readable.
users.users.audiobookshelf = {
extraGroups = [ "media" ];
};
systemd.services.audiobookshelfd.serviceConfig.Group = lib.mkForce "media";
systemd.services.audiobookshelfd.serviceConfig.UMask = lib.mkForce "0027";
# We backup the whole audiobookshelf directory and set permissions for the backup user accordingly.
users.groups.audiobookshelf.members = [ "backup" ];
users.groups.media.members = [ "backup" ];
} {
systemd.services.audiobookshelfd.serviceConfig = cfg.extraServiceConfig;
}]);

12
modules/services/deluge.nix
View file

@ -251,6 +251,7 @@ in
'';
readOnly = true;
default = {
user = "deluge";
sourceDirectories = [
cfg.dataDir
];
@ -373,17 +374,6 @@ in
inherit (cfg) authEndpoint;
}))
];
# We want deluge to create files in the media group and to make those files group readable.
users.users.deluge = {
extraGroups = [ "media" ];
};
systemd.services.deluged.serviceConfig.Group = lib.mkForce "media";
systemd.services.deluged.serviceConfig.UMask = lib.mkForce "0027";
# We backup the whole deluge directory and set permissions for the backup user accordingly.
users.groups.deluge.members = [ "backup" ];
users.groups.media.members = [ "backup" ];
} {
systemd.services.deluged.serviceConfig = cfg.extraServiceConfig;
} (lib.mkIf (config.shb.deluge.prometheusScraperPasswordFile != null) {

5
modules/services/grocy.nix
View file

@ -80,6 +80,7 @@ in
'';
readOnly = true;
default = {
user = "grocy";
sourceDirectories = [
cfg.dataDir
];
@ -115,10 +116,6 @@ in
sslCertificate = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.cert;
sslCertificateKey = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.key;
};
# We backup the whole grocy directory and set permissions for the backup user accordingly.
users.groups.grocy.members = [ "backup" ];
users.groups.media.members = [ "backup" ];
} {
systemd.services.grocyd.serviceConfig = cfg.extraServiceConfig;
}]);

1
modules/services/hledger.nix
View file

@ -72,6 +72,7 @@ in
'';
readOnly = true;
default = {
user = "hledger";
sourceDirectories = [
cfg.dataDir
];

18
modules/services/home-assistant.nix
View file

@ -154,6 +154,7 @@ in
'';
readOnly = true;
default = {
user = "hass";
# No need for backup hooks as we use an hourly automation job in home assistant directly with a cron job.
sourceDirectories = [
"/var/lib/hass/backups"
@ -322,22 +323,5 @@ in
"f ${config.services.home-assistant.configDir}/scenes.yaml 0755 hass hass"
"f ${config.services.home-assistant.configDir}/scripts.yaml 0755 hass hass"
];
# Adds the "backup" user to the "hass" group.
users.groups.hass = {
members = [ "backup" ];
};
# This allows the "backup" user, member of the "backup" group, to access what's inside the home
# folder, which is needed for accessing the "backups" folder. It allows to read (r), enter the
# directory (x) but not modify what's inside.
users.users.hass.homeMode = "0750";
systemd.services.home-assistant.serviceConfig = {
# This allows all members of the "hass" group to read files, list directories and enter
# directories created by the home-assistant service. This is needed for the "backup" user,
# member of the "hass" group, to backup what is inside the "backup/" folder.
UMask = lib.mkForce "0027";
};
};
}

19
modules/services/jellyfin.nix
View file

@ -138,6 +138,7 @@ in
'';
readOnly = true;
default = {
user = "jellyfin";
sourceDirectories = [
"/var/lib/jellyfin"
];
@ -153,16 +154,6 @@ in
allowedUDPPorts = [ 1900 7359 ];
};
users.groups = {
media = {
name = "media";
members = [ "jellyfin" ];
};
jellyfin = {
members = [ "backup" ];
};
};
services.nginx.enable = true;
# Take advice from https://jellyfin.org/docs/general/networking/nginx/ and https://nixos.wiki/wiki/Plex
@ -432,13 +423,5 @@ in
redirect_uris = [ "https://${cfg.subdomain}.${cfg.domain}/sso/OID/r/${cfg.sso.provider}" ];
}
];
# For backup
systemd.services.jellyfin.serviceConfig = {
# Setup permissions needed for backups, as the backup user is member of the jellyfin group.
UMask = lib.mkForce "0027";
StateDirectoryMode = lib.mkForce "0750";
};
};
}

11
modules/services/nextcloud-server.nix
View file

@ -515,6 +515,7 @@ in
'';
readOnly = true;
default = {
user = "nextcloud";
sourceDirectories = [
cfg.dataDir
];
@ -568,12 +569,6 @@ in
};
};
# users.groups = {
# nextcloud = {
# members = [ "backup" ];
# };
# };
# LDAP is manually configured through
# https://github.com/lldap/lldap/blob/main/example_configs/nextcloud.md, see also
# https://docs.nextcloud.com/server/latest/admin_manual/configuration_user/user_auth_ldap.html
@ -708,10 +703,6 @@ in
services.postgresql.settings = lib.mkIf (! (isNull cfg.postgresSettings)) cfg.postgresSettings;
systemd.services.phpfpm-nextcloud.serviceConfig = {
# Setup permissions needed for backups, as the backup user is member of the jellyfin group.
UMask = lib.mkForce "0027";
};
systemd.services.phpfpm-nextcloud.preStart = ''
mkdir -p /var/log/xdebug; chown -R nextcloud: /var/log/xdebug
'';

12
modules/services/vaultwarden.nix
View file

@ -132,6 +132,7 @@ in
'';
readOnly = true;
default = {
user = "vaultwarden";
sourceDirectories = [
dataFolder
];
@ -224,17 +225,6 @@ in
passwordFile = builtins.toString cfg.databasePasswordFile;
}
];
systemd.services.vaultwarden.serviceConfig.UMask = lib.mkForce "0027";
# systemd.services.vaultwarden.serviceConfig.Group = lib.mkForce "media";
users.users.vaultwarden = {
extraGroups = [ "media" ];
};
users.groups.vaultwarden = {
members = [ "backup" ];
};
# TODO: make this work.
# It does not work because it leads to infinite recursion.
# ${cfg.mount}.path = dataFolder;

374
test/blocks/restic.nix
View file

@ -9,240 +9,184 @@ let
../../modules/blocks/restic.nix
];
commonTestScript = ''
from dictdiffer import diff
commonTest = user: pkgs.testers.runNixOSTest {
name = "restic_backupAndRestore_${user}";
def list_files(dir):
files_and_content = {}
nodes.machine = {
imports = ( testLib.baseImports pkgs' ) ++ [
../../modules/blocks/restic.nix
];
files = machine.succeed(f"""
find {dir} -type f
""").split("\n")[:-1]
shb.restic.instances."testinstance" = {
enable = true;
for f in files:
content = machine.succeed(f"""
cat {f}
""").strip()
files_and_content[f] = content
passphraseFile = pkgs.writeText "passphrase" "PassPhrase";
return files_and_content
sourceDirectories = [
"/opt/files/A"
"/opt/files/B"
];
def assert_files(dir, files):
result = list(diff(list_files(dir), files))
if len(result) > 0:
raise Exception("Unexpected files:", result)
user = user;
with subtest("Create initial content"):
machine.succeed("""
mkdir -p /opt/files/A
mkdir -p /opt/files/B
repositories = [
{
path = "/opt/repos/A";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
# Those are not needed by the repository but are still included
# so we can test them in the hooks section.
secrets = {
A.source = "/run/secrets/A";
B.source = "/run/secrets/B";
};
}
{
path = "/opt/repos/B";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
}
];
echo repoA_fileA_1 > /opt/files/A/fileA
echo repoA_fileB_1 > /opt/files/A/fileB
echo repoB_fileA_1 > /opt/files/B/fileA
echo repoB_fileB_1 > /opt/files/B/fileB
hooks.before_backup = [''
echo $RUNTIME_DIRECTORY
if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
if ! [ -f /run/secrets_restic/restic-backups-testinstance_opt_repos_A ]; then
exit 10
fi
if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
echo "A:$A"
exit 11
fi
if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
echo "B:$B"
exit 12
fi
fi
''];
};
};
# chown :backup -R /opt/files
""")
extraPythonPackages = p: [ p.dictdiffer ];
skipTypeCheck = true;
assert_files("/opt/files", {
'/opt/files/B/fileA': 'repoB_fileA_1',
'/opt/files/B/fileB': 'repoB_fileB_1',
'/opt/files/A/fileA': 'repoA_fileA_1',
'/opt/files/A/fileB': 'repoA_fileB_1',
})
testScript = ''
from dictdiffer import diff
with subtest("First backup in repo A"):
machine.succeed("systemctl start restic-backups-testinstance_opt_repos_A")
def list_files(dir):
files_and_content = {}
with subtest("New content"):
machine.succeed("""
echo repoA_fileA_2 > /opt/files/A/fileA
echo repoA_fileB_2 > /opt/files/A/fileB
echo repoB_fileA_2 > /opt/files/B/fileA
echo repoB_fileB_2 > /opt/files/B/fileB
""")
files = machine.succeed(f"""
find {dir} -type f
""").split("\n")[:-1]
assert_files("/opt/files", {
'/opt/files/B/fileA': 'repoB_fileA_2',
'/opt/files/B/fileB': 'repoB_fileB_2',
'/opt/files/A/fileA': 'repoA_fileA_2',
'/opt/files/A/fileB': 'repoA_fileB_2',
})
for f in files:
content = machine.succeed(f"""
cat {f}
""").strip()
files_and_content[f] = content
with subtest("Second backup in repo B"):
machine.succeed("systemctl start restic-backups-testinstance_opt_repos_B")
return files_and_content
with subtest("Delete content"):
machine.succeed("""
rm -r /opt/files/A /opt/files/B
""")
def assert_files(dir, files):
result = list(diff(list_files(dir), files))
if len(result) > 0:
raise Exception("Unexpected files:", result)
assert_files("/opt/files", {})
with subtest("Create secrets"):
print(machine.succeed("""
mkdir -p /run/secrets/
with subtest("Restore initial content from repo A"):
machine.succeed("""
restic-testinstance_opt_repos_A restore latest -t /
""")
echo secretA > /run/secrets/A
echo secretB > /run/secrets/B
assert_files("/opt/files", {
'/opt/files/B/fileA': 'repoB_fileA_1',
'/opt/files/B/fileB': 'repoB_fileB_1',
'/opt/files/A/fileA': 'repoA_fileA_1',
'/opt/files/A/fileB': 'repoA_fileB_1',
})
chown root:keys -R /run/secrets
find /run/secrets -type d -exec chmod u=rwx,g=rx,o=x '{}' ';'
find /run/secrets -type f -exec chmod u=r,g=r,o= '{}' ';'
ls -l /run/secrets
"""))
with subtest("Restore initial content from repo B"):
machine.succeed("""
restic-testinstance_opt_repos_B restore latest -t /
""")
with subtest("Create initial content"):
machine.succeed("""
mkdir -p /opt/files/A
mkdir -p /opt/files/B
assert_files("/opt/files", {
'/opt/files/B/fileA': 'repoB_fileA_2',
'/opt/files/B/fileB': 'repoB_fileB_2',
'/opt/files/A/fileA': 'repoA_fileA_2',
'/opt/files/A/fileB': 'repoA_fileB_2',
})
'';
echo repoA_fileA_1 > /opt/files/A/fileA
echo repoA_fileB_1 > /opt/files/A/fileB
echo repoB_fileA_1 > /opt/files/B/fileA
echo repoB_fileB_1 > /opt/files/B/fileB
chown ${user}: -R /opt/files
chmod go-rwx -R /opt/files
""")
assert_files("/opt/files", {
'/opt/files/B/fileA': 'repoB_fileA_1',
'/opt/files/B/fileB': 'repoB_fileB_1',
'/opt/files/A/fileA': 'repoA_fileA_1',
'/opt/files/A/fileB': 'repoA_fileB_1',
})
with subtest("First backup in repo A"):
machine.succeed("systemctl start restic-backups-testinstance_opt_repos_A")
with subtest("New content"):
machine.succeed("""
echo repoA_fileA_2 > /opt/files/A/fileA
echo repoA_fileB_2 > /opt/files/A/fileB
echo repoB_fileA_2 > /opt/files/B/fileA
echo repoB_fileB_2 > /opt/files/B/fileB
""")
assert_files("/opt/files", {
'/opt/files/B/fileA': 'repoB_fileA_2',
'/opt/files/B/fileB': 'repoB_fileB_2',
'/opt/files/A/fileA': 'repoA_fileA_2',
'/opt/files/A/fileB': 'repoA_fileB_2',
})
with subtest("Second backup in repo B"):
machine.succeed("systemctl start restic-backups-testinstance_opt_repos_B")
with subtest("Delete content"):
machine.succeed("""
rm -r /opt/files/A /opt/files/B
""")
assert_files("/opt/files", {})
with subtest("Restore initial content from repo A"):
machine.succeed("""
restic-testinstance_opt_repos_A restore latest -t /
""")
assert_files("/opt/files", {
'/opt/files/B/fileA': 'repoB_fileA_1',
'/opt/files/B/fileB': 'repoB_fileB_1',
'/opt/files/A/fileA': 'repoA_fileA_1',
'/opt/files/A/fileB': 'repoA_fileB_1',
})
with subtest("Restore initial content from repo B"):
machine.succeed("""
restic-testinstance_opt_repos_B restore latest -t /
""")
assert_files("/opt/files", {
'/opt/files/B/fileA': 'repoB_fileA_2',
'/opt/files/B/fileB': 'repoB_fileB_2',
'/opt/files/A/fileA': 'repoA_fileA_2',
'/opt/files/A/fileB': 'repoA_fileB_2',
})
'';
};
in
{
backupAndRestoreRoot = pkgs.testers.runNixOSTest {
name = "restic_backupAndRestore";
nodes.machine = {
imports = ( testLib.baseImports pkgs' ) ++ [
../../modules/blocks/restic.nix
];
shb.restic = {
user = "root";
group = "root";
};
shb.restic.instances."testinstance" = {
enable = true;
passphraseFile = pkgs.writeText "passphrase" "PassPhrase";
sourceDirectories = [
"/opt/files/A"
"/opt/files/B"
];
repositories = [
{
path = "/opt/repos/A";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
# Those are not needed by the repository but are still included
# so we can test them in the hooks section.
secrets = {
A.source = pkgs.writeText "A" "secretA";
B.source = pkgs.writeText "B" "secretB";
};
}
{
path = "/opt/repos/B";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
}
];
hooks.before_backup = [''
echo $RUNTIME_DIRECTORY
if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
if ! [ -f /run/secrets/restic/restic-backups-testinstance_opt_repos_A ]; then
exit 10
fi
if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
echo "A:$A"
exit 11
fi
if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
echo "A:$A"
exit 12
fi
fi
''];
};
};
extraPythonPackages = p: [ p.dictdiffer ];
skipTypeCheck = true;
testScript = commonTestScript;
};
backupAndRestoreUser = pkgs.testers.runNixOSTest {
name = "restic_backupAndRestore";
nodes.machine = {
imports = ( testLib.baseImports pkgs' ) ++ [
../../modules/blocks/restic.nix
];
shb.restic = {
user = "backup";
group = "backup";
};
shb.restic.instances."testinstance" = {
enable = true;
passphraseFile = pkgs.writeText "passphrase" "PassPhrase";
sourceDirectories = [
"/opt/files/A"
"/opt/files/B"
];
repositories = [
{
path = "/opt/repos/A";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
# Those are not needed by the repository but are still included
# so we can test them in the hooks section.
secrets = {
A.source = pkgs.writeText "A" "secretA";
B.source = pkgs.writeText "B" "secretB";
};
}
{
path = "/opt/repos/B";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
}
];
hooks.before_backup = [''
echo $RUNTIME_DIRECTORY
if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
if ! [ -f /run/secrets/restic/restic-backups-testinstance_opt_repos_A ]; then
exit 10
fi
if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
echo "A:$A"
exit 11
fi
if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
echo "A:$A"
exit 12
fi
fi
''];
};
};
extraPythonPackages = p: [ p.dictdiffer ];
skipTypeCheck = true;
testScript = commonTestScript;
};
backupAndRestoreRoot = commonTest "root";
backupAndRestoreUser = commonTest "nobody";
}

2
test/modules/arr.nix
View file

@ -90,7 +90,6 @@ in
ssl = null;
}
];
users.groups.radarr.members = [ "backup" ];
services.nginx.enable = true;
services.bazarr = {};
services.jackett = {};
@ -156,7 +155,6 @@ in
ssl = null;
}
];
users.groups.radarr.members = [ "backup" ];
services.nginx.enable = true;
services.bazarr = {};
services.jackett = {};