restructure backups

This commit is contained in:
2025-10-18 13:51:23 -05:00
parent f597616843
commit 643ab0958c
22 changed files with 385 additions and 262 deletions

View File

@@ -92,7 +92,7 @@ in
icon = "di:${service}"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -94,7 +94,7 @@ in
icon = "di:${service}"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -97,7 +97,7 @@ in
icon = "di:${service}"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -95,7 +95,7 @@ in
icon = "di:${service}"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -98,7 +98,7 @@ in
icon = "di:${service}"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -95,7 +95,7 @@ in
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -37,9 +37,15 @@ in
type = lib.types.str;
description = "base domain used for reverse proxy";
};
baks = lib.mkOption {
type = lib.types.attrsOf (lib.types.attrsOf (lib.types.listOf lib.types.path));
default = {};
description = "backup jobs, nested attribute sets should be <bak_name> = paths [<list_of_paths>]";
};
backup_repo = lib.mkOption {
type = lib.types.path;
default = "/holocron/archives/homelab";
description = "path to take daily backups to with borg!";
};
};
@@ -78,7 +84,7 @@ in
};
};
# backups with borg
# backups homelab with borg
services.borgbackup.jobs.homelab = {
archiveBaseName = "homelab";
repo = cfg.backup_repo;
@@ -86,9 +92,10 @@ in
compression = "auto,zstd";
startAt = "daily";
group = "archives";
encryption.mode = "repokey";
encryption.mode = "repokey-blake2";
encryption.passCommand = "cat ${config.sops.secrets."borg_passwd".path}";
extraArgs = ["--verbose" "--list" "--filter AME" "--stats" "--show-rc" "--umask 0007"];
extraArgs = ["--verbose" "--show-rc" "--umask" "0007"];
extraCreateArgs = ["--list" "--stats" "--filter" "AME"];
prune.keep = {
within = "1d"; # Keep all archives from the last day
daily = 7;

View File

@@ -125,7 +125,7 @@ in
};
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -313,7 +313,7 @@ in
};
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = {
paths = [ cfg.data_dir ];
};

View File

@@ -109,7 +109,7 @@ in
icon = "di:${nixservice}"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -92,7 +92,7 @@ in
};
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -123,7 +123,7 @@ in
};
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -108,7 +108,7 @@ in {
];
# add to backups
system.backups.baks = {
homelab.baks = {
#${service} = {paths = [cfg.data_dir "/var/lib/redis-immich" "/var/backup/postgresql/immich.sql.zstd"];};
${service} = {paths = [cfg.data_dir "/var/backup/postgresql/immich.sql.zstd"];};
};

View File

@@ -93,7 +93,7 @@ in
icon = "di:${service}"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -1,17 +1,32 @@
{ pkgs, config, lib, ... }:
let
{
pkgs,
config,
lib,
...
}: let
service = "minecraft_recpro";
cfg = config.gameservers.${service};
sec = config.sops.secrets;
servers = {
velocity = { data_dir = "/var/lib/gameservers/minecraft_recpro/velocity"; ram = "2G"; };
smp = { data_dir = "/var/lib/gameservers/minecraft_recpro/smp"; ram = "12G"; };
superflat = { data_dir = "/var/lib/gameservers/minecraft_recpro/superflat"; ram = "4G"; };
bento = { data_dir = "/var/lib/gameservers/minecraft_recpro/bento"; ram = "2G"; };
velocity = {
data_dir = "/var/lib/gameservers/minecraft_recpro/velocity";
db_dumb_dir = "/var/backup/mysql/${service}_db.zst";
ram = "2G";
};
in
{
smp = {
data_dir = "/var/lib/gameservers/minecraft_recpro/smp";
ram = "12G";
};
superflat = {
data_dir = "/var/lib/gameservers/minecraft_recpro/superflat";
ram = "4G";
};
bento = {
data_dir = "/var/lib/gameservers/minecraft_recpro/bento";
ram = "2G";
};
};
in {
options.gameservers.${service} = {
enable = lib.mkEnableOption "enables ${service}";
url = lib.mkOption {
@@ -38,10 +53,14 @@ in
type = lib.types.nullOr lib.types.str;
default = "velocity";
};
backup_repo = lib.mkOption {
type = lib.types.path;
default = "/holocron/archives/gameservers/minecraft/recpro_stack";
description = "path to take hourly backups to with borg!";
};
};
config = lib.mkIf cfg.enable {
# declare ${service} group
users.groups.minecraft = {gid = lib.mkForce cfg.ids;};
@@ -55,12 +74,15 @@ in
extraGroups = [];
};
systemd.tmpfiles.rules = lib.attrsets.mapAttrsToList (name: cfg:
"d ${cfg.data_dir} 0770 minecraft minecraft -"
) servers;
systemd.tmpfiles.rules =
lib.attrsets.mapAttrsToList (
name: cfg: "d ${cfg.data_dir} 0770 minecraft minecraft -"
)
servers;
# Create a systemd service per server running in tmux
systemd.services = lib.attrsets.mapAttrs (name: srv: {
systemd.services =
lib.attrsets.mapAttrs (name: srv: {
description = "minecraft_recpro: ${name}";
after = ["network.target"];
wants = ["network.target"];
@@ -74,7 +96,8 @@ in
KillMode = "process";
};
wantedBy = ["multi-user.target"];
}) servers;
})
servers;
environment.systemPackages = with pkgs; [openjdk21 mcrcon];
@@ -109,17 +132,29 @@ in
owner = "mysql";
group = "mysql";
};
};
# add to backups
system.backups.gameserver_baks = lib.listToAttrs (
lib.mapAttrsToList (srv_name: cfg:
{
name = srv_name; # attribute key
value = { paths = [ cfg.data_dir "/var/backup/mysql/${service}_db.zst" ]; }; # attribute value
}
) servers
);
# backups minecraft_recpro with borg!
services.borgbackup.jobs.${service} =
lib.mapAttrs (name: srv: {
archiveBaseName = "${name}";
repo = cfg.backup_repo;
paths = [srv.data_dir] ++ lib.optionals (srv ? db_dump_dir) [srv.db_path];
compression = "auto,zstd";
startAt = "*-*-* *:00:00";
group = "archives";
encryption.mode = "repokey-blake2";
encryption.passCommand = "cat ${config.sops.secrets."borg_passwd".path}";
extraArgs = ["--verbose" "--show-rc" "--umask" "0007"];
extraCreateArgs = ["--list" "--stats" "--filter" "AME"];
prune.keep = {
within = "1d"; # Keep all archives from the last day
hourly = 24;
daily = 7;
weekly = 12;
monthly = -1; # Keep at least one archive for each month
};
})
servers;
};
}

View File

@@ -94,13 +94,5 @@ in {
programs.zsh.interactiveShellInit = ''
/etc/motd
'';
#environment.loginShellInit = ''
# if [ -x /etc/motd ]; then
# /etc/motd
# else
# cat /etc/motd
# fi
#'';
};
}

View File

@@ -131,7 +131,7 @@ in
icon = "di:${service}"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -93,7 +93,7 @@ in
icon = "di:${service}"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -113,7 +113,7 @@ in
};
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -101,7 +101,7 @@ in
icon = "di:yac-reader"; }];
# add to backups
system.backups.baks = {
homelab.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
};

View File

@@ -58,193 +58,6 @@ in {
config = lib.mkIf (cfg.enable && cfg.baks != {}) {
# create and or set perms for repo dirs
systemd.tmpfiles.rules = [
"d ${cfg.repo} 2770 root archives - -"
"d ${cfg.gameserver_repo} 2770 root archives - -"
];
# create servie to backup services
systemd.services.backups = {
description = "backup services with borg!";
path = [pkgs.borgbackup];
serviceConfig = {
Type = "oneshot";
User = "root";
Group = "archives"; # make perms shake out
UMask = "0007"; # make perms shake out
# the actual script borg is using
ExecStart = pkgs.writeShellScript "borg-backup" ''
backup() {
set -euo pipefail
export BORG_PASSPHRASE="$(cat ${cfg.passwd_file})"
export BORG_REPO="${cfg.repo}"
timestamp="$(date +'%Y-%m-%d_%H:%M:%S')"
mode=split
# init repo in needed
if ! borg info "$BORG_REPO" >/dev/null 2>&1; then
echo "Initializing Borg repo at $BORG_REPO"
borg init --encryption=repokey "$BORG_REPO"
fi
borg break-lock "$BORG_REPO" || true
echo "starting backup at $timestamp"
if [ "$mode" = "split" ]; then
# loop for each backup
${lib.concatStringsSep "\n\n" (lib.mapAttrsToList (
bak_name: bak_paths: ''
echo "------------ Backing up ${bak_name} ------------"
archive="$timestamp-${bak_name}"
echo "backing up: ${lib.concatStringsSep " " bak_paths.paths} $archive"
borg create \
--verbose \
--filter AME \
--list \
--stats \
--show-rc \
--compression lz4 \
"$BORG_REPO::$archive" \
${lib.concatStringsSep " " bak_paths.paths}
echo "pruning old backups for ${bak_name}..."
borg prune -v --list "$BORG_REPO" \
--glob-archives "*-${bak_name}" \
--keep-daily=7 \
--keep-weekly=52 \
--keep-monthly=-1
echo "backup run complete at \"$BORG_REPO::$archive\""
''
)
cfg.baks)}
exit 0
else
# flatten all paths from cfg.baks into one big list
all_paths="${
lib.concatStringsSep " "
(lib.flatten
(lib.mapAttrsToList (_: bak: bak.paths) cfg.baks))
}"
borg create \
--verbose \
--filter AME \
--list \
--stats \
--show-rc \
--compression lzma,9 \
"$BORG_REPO::$timestamp-${toString config.networking.hostName}" \
$all_paths
echo "pruning old backups for ${toString config.networking.hostName}..."
borg prune -v --list "$BORG_REPO" \
--glob-archives "*-${toString config.networking.hostName}" \
--keep-daily=7 \
--keep-weekly=52 \
--keep-monthly=-1
echo "backup run complete at \"$BORG_REPO::${toString config.networking.hostName}\""
exit 0
fi
}
start_time=$(date +%s)
backup
end_time=$(date +%s)
exec_time=$((end_time - start_time))
cpu_usage=$(top -bn1 | grep "Cpu(s)" | sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | awk '{print 100 - $1"%"}')
echo ""
echo "backup stats:"
echo "exec time: $exec_time"
echo "cpu usage: $cpu_usage"
'';
};
};
# create timer to run backups daily
systemd.timers.backups = {
description = "daily borg backup timer";
wantedBy = ["timers.target"];
timerConfig = {
OnCalendar = "04:00";
Persistent = true;
};
};
# create servie to backup gameservers (back these up hourly)
systemd.services.gameserver_backups = {
description = "backup services with borg!";
path = [pkgs.borgbackup];
serviceConfig = {
Type = "oneshot";
User = "root";
Group = "archives"; # make perms shake out
UMask = "0007"; # make perms shake out
# the actual script borg is using
ExecStart = pkgs.writeShellScript "borg-gameserver_backup" ''
backup() {
set -euo pipefail
export BORG_PASSPHRASE="$(cat ${cfg.passwd_file})"
export BORG_REPO="${cfg.gameserver_repo}"
timestamp="$(date +'%Y-%m-%d_%H:%M:%S')"
# init repo in needed
if ! borg info "$BORG_REPO" >/dev/null 2>&1; then
echo "Initializing Borg repo at $BORG_REPO"
borg init --encryption=repokey "$BORG_REPO"
fi
borg break-lock "$BORG_REPO" || true
echo "starting backup at $timestamp"
# loop for each backup
${lib.concatStringsSep "\n\n" (lib.mapAttrsToList (
bak_name: bak_paths: ''
echo "------------ Backing up ${bak_name} ------------"
archive="$timestamp-${bak_name}"
echo "backing up: ${lib.concatStringsSep " " bak_paths.paths} $archive"
borg create \
--verbose \
--filter AME \
--list \
--stats \
--show-rc \
--compression lz4 \
"$BORG_REPO::$archive" \
${lib.concatStringsSep " " bak_paths.paths}
echo "pruning old backups for ${bak_name}..."
borg prune -v --list "$BORG_REPO" \
--glob-archives "*-${bak_name}" \
--keep-hourly=24 \
--keep-daily=7 \
--keep-weekly=12 \
--keep-monthly=12
echo "backup run complete at \"$BORG_REPO::$archive\""
''
)
cfg.gameserver_baks)}
exit 0
}
start_time=$(date +%s)
backup
end_time=$(date +%s)
exec_time=$((end_time - start_time))
cpu_usage=$(top -bn1 | grep "Cpu(s)" | sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | awk '{print 100 - $1"%"}')
echo ""
echo "backup stats:"
echo "exec time: $exec_time"
echo "cpu usage: $cpu_usage"
'';
};
};
# create timer to run backups daily
systemd.timers.gameserver_backups = {
description = "daily borg backup timer";
wantedBy = ["timers.target"];
timerConfig = {
OnCalendar = "*-*-* *:00:00"; # every hour, at :01 (one min after db dump)
Persistent = true;
};
};
# db backups
services.mysqlBackup = lib.mkIf config.services.mysql.enable {
# mc servers use this
@@ -265,9 +78,6 @@ in {
#databases = config.services.postgresql.ensureDatabases; # set to all databases defined in esure databases
};
# install borg binary
environment.systemPackages = with pkgs; [borgbackup tree];
# declare secret for repo password
sops.secrets = {
"borg_passwd" = {

View File

@@ -0,0 +1,279 @@
{
config,
lib,
pkgs,
...
}:
/*
this module enables a backup script made with borg!
to use import & set the options below
to declare a backup add the following code
to a module and it will backup all listed paths
in a borg archive to the specified repo
| <3yy> |
V V
system.backups.baks = {
${service} = { paths = [ cfg.data_dir ]; };
};
*/
let
cfg = config.system.backups;
sec = config.sops.secrets;
borg = "${pkgs.borgbackup}/bin/borg";
in {
options.system.backups = {
enable = lib.mkEnableOption "enables backups with borg";
baks = lib.mkOption {
type = lib.types.attrsOf (lib.types.attrsOf (lib.types.listOf lib.types.path));
default = {};
description = "backup jobs, nested attribute sets should be <bak_name> = paths [<list_of_paths>]";
};
gameserver_baks = lib.mkOption {
type = lib.types.attrsOf (lib.types.attrsOf (lib.types.listOf lib.types.path));
default = {};
description = "backup jobs for game servers, nested attribute sets should be <bak_name> = paths [<list_of_paths>]";
};
repo = lib.mkOption {
type = lib.types.path;
default = "/holocron/archives/devices/snowbelle";
description = "borg repository path";
};
gameserver_repo = lib.mkOption {
type = lib.types.path;
default = "/holocron/archives/gameservers/borg";
description = "borg repository path";
};
passwd_file = lib.mkOption {
type = lib.types.path;
default = sec."borg_passwd".path;
description = "borg repository passphrase file";
};
mode = lib.mkOption {
type = lib.types.str;
default = "split"; # "all"
description = "choice between creating one archive of all paths or one archive per service";
};
};
config = lib.mkIf (cfg.enable && cfg.baks != {}) {
# create and or set perms for repo dirs
systemd.tmpfiles.rules = [
"d ${cfg.repo} 2770 root archives - -"
"d ${cfg.gameserver_repo} 2770 root archives - -"
];
# create servie to backup services
systemd.services.backups = {
description = "backup services with borg!";
path = [pkgs.borgbackup];
serviceConfig = {
Type = "oneshot";
User = "root";
Group = "archives"; # make perms shake out
UMask = "0007"; # make perms shake out
# the actual script borg is using
ExecStart = pkgs.writeShellScript "borg-backup" ''
backup() {
set -euo pipefail
export BORG_PASSPHRASE="$(cat ${cfg.passwd_file})"
export BORG_REPO="${cfg.repo}"
timestamp="$(date +'%Y-%m-%d_%H:%M:%S')"
mode=split
# init repo in needed
if ! borg info "$BORG_REPO" >/dev/null 2>&1; then
echo "Initializing Borg repo at $BORG_REPO"
borg init --encryption=repokey "$BORG_REPO"
fi
borg break-lock "$BORG_REPO" || true
echo "starting backup at $timestamp"
if [ "$mode" = "split" ]; then
# loop for each backup
${lib.concatStringsSep "\n\n" (lib.mapAttrsToList (
bak_name: bak_paths: ''
echo "------------ Backing up ${bak_name} ------------"
archive="$timestamp-${bak_name}"
echo "backing up: ${lib.concatStringsSep " " bak_paths.paths} $archive"
borg create \
--verbose \
--filter AME \
--list \
--stats \
--show-rc \
--compression lz4 \
"$BORG_REPO::$archive" \
${lib.concatStringsSep " " bak_paths.paths}
echo "pruning old backups for ${bak_name}..."
borg prune -v --list "$BORG_REPO" \
--glob-archives "*-${bak_name}" \
--keep-daily=7 \
--keep-weekly=52 \
--keep-monthly=-1
echo "backup run complete at \"$BORG_REPO::$archive\""
''
)
cfg.baks)}
exit 0
else
# flatten all paths from cfg.baks into one big list
all_paths="${
lib.concatStringsSep " "
(lib.flatten
(lib.mapAttrsToList (_: bak: bak.paths) cfg.baks))
}"
borg create \
--verbose \
--filter AME \
--list \
--stats \
--show-rc \
--compression lzma,9 \
"$BORG_REPO::$timestamp-${toString config.networking.hostName}" \
$all_paths
echo "pruning old backups for ${toString config.networking.hostName}..."
borg prune -v --list "$BORG_REPO" \
--glob-archives "*-${toString config.networking.hostName}" \
--keep-daily=7 \
--keep-weekly=52 \
--keep-monthly=-1
echo "backup run complete at \"$BORG_REPO::${toString config.networking.hostName}\""
exit 0
fi
}
start_time=$(date +%s)
backup
end_time=$(date +%s)
exec_time=$((end_time - start_time))
cpu_usage=$(top -bn1 | grep "Cpu(s)" | sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | awk '{print 100 - $1"%"}')
echo ""
echo "backup stats:"
echo "exec time: $exec_time"
echo "cpu usage: $cpu_usage"
'';
};
};
# create timer to run backups daily
systemd.timers.backups = {
description = "daily borg backup timer";
wantedBy = ["timers.target"];
timerConfig = {
OnCalendar = "04:00";
Persistent = true;
};
};
# create servie to backup gameservers (back these up hourly)
systemd.services.gameserver_backups = {
description = "backup services with borg!";
path = [pkgs.borgbackup];
serviceConfig = {
Type = "oneshot";
User = "root";
Group = "archives"; # make perms shake out
UMask = "0007"; # make perms shake out
# the actual script borg is using
ExecStart = pkgs.writeShellScript "borg-gameserver_backup" ''
backup() {
set -euo pipefail
export BORG_PASSPHRASE="$(cat ${cfg.passwd_file})"
export BORG_REPO="${cfg.gameserver_repo}"
timestamp="$(date +'%Y-%m-%d_%H:%M:%S')"
# init repo in needed
if ! borg info "$BORG_REPO" >/dev/null 2>&1; then
echo "Initializing Borg repo at $BORG_REPO"
borg init --encryption=repokey "$BORG_REPO"
fi
borg break-lock "$BORG_REPO" || true
echo "starting backup at $timestamp"
# loop for each backup
${lib.concatStringsSep "\n\n" (lib.mapAttrsToList (
bak_name: bak_paths: ''
echo "------------ Backing up ${bak_name} ------------"
archive="$timestamp-${bak_name}"
echo "backing up: ${lib.concatStringsSep " " bak_paths.paths} $archive"
borg create \
--verbose \
--filter AME \
--list \
--stats \
--show-rc \
--compression lz4 \
"$BORG_REPO::$archive" \
${lib.concatStringsSep " " bak_paths.paths}
echo "pruning old backups for ${bak_name}..."
borg prune -v --list "$BORG_REPO" \
--glob-archives "*-${bak_name}" \
--keep-hourly=24 \
--keep-daily=7 \
--keep-weekly=12 \
--keep-monthly=12
echo "backup run complete at \"$BORG_REPO::$archive\""
''
)
cfg.gameserver_baks)}
exit 0
}
start_time=$(date +%s)
backup
end_time=$(date +%s)
exec_time=$((end_time - start_time))
cpu_usage=$(top -bn1 | grep "Cpu(s)" | sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | awk '{print 100 - $1"%"}')
echo ""
echo "backup stats:"
echo "exec time: $exec_time"
echo "cpu usage: $cpu_usage"
'';
};
};
# create timer to run backups daily
systemd.timers.gameserver_backups = {
description = "daily borg backup timer";
wantedBy = ["timers.target"];
timerConfig = {
OnCalendar = "*-*-* *:00:00"; # every hour, at :01 (one min after db dump)
Persistent = true;
};
};
# db backups
services.mysqlBackup = lib.mkIf config.services.mysql.enable {
# mc servers use this
enable = true;
location = "/var/backup/mysql";
user = "root";
calendar = "*-*-* *:59:00";
compressionAlg = "zstd";
databases = config.services.mysql.ensureDatabases; # set to all databases defined in esure databases
};
services.postgresqlBackup = lib.mkIf config.services.postgresql.enable {
# immich uses this
enable = true;
location = "/var/backup/postgresql";
compression = "zstd"; # optional: "xz", "zstd", "none"
startAt = "03:58";
databases = ["immich"]; # set to all databases defined in esure databases
#databases = config.services.postgresql.ensureDatabases; # set to all databases defined in esure databases
};
# install borg binary
environment.systemPackages = with pkgs; [borgbackup tree];
# declare secret for repo password
sops.secrets = {
"borg_passwd" = {
owner = "root";
group = "root";
};
};
};
}