toph/dot.nix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Enhances backup orchestration and notifications

Browse source 
• Move lazydocker pkg to docker.nix
• Extends backup service generator with optional scheduling, verbose logging, and new mkAppriseUrl
• Refactors backup configurations and renames files for clarity
• Introduces backup chain orchestration for smoother maintenance
• Updates Apprise URL generation and removes deprecated secret spec functions
This commit is contained in:
Chris Toph 2025-05-04 17:17:33 -04:00
parent 63fbfe8426
commit 6de78e75e6
7 changed files with 270 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
hosts/common/optional/docker.nix
View file

@ -7,4 +7,8 @@
};
oci-containers.backend = "docker";
};
environment.systemPackages = with pkgs; [
lazydocker
];
}

67
hosts/nixos/cloud/config/borg.nix → hosts/nixos/cloud/config/backups/borg.nix
View file

@ -60,7 +60,7 @@ let
STATS=$(cat ${logFile}.stats || echo "No stats available")
'';
# Unified backup service generator
# Unified backup service generator with optional features
mkBorgBackupService =
{
name,
@ -70,8 +70,25 @@ let
keepDaily,
keepWeekly,
keepMonthly,
schedule,
schedule ? null,
enableNotifications ? true,
verbose ? false,
}:
let
maybeCreateTimer = lib.optionalAttrs (schedule != null) {
timers."backup-${name}" = {
description = "Timer for ${title} Backup";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = schedule;
Persistent = true;
RandomizedDelaySec = "5min";
};
};
};
logPrefix = if verbose then "set -x;" else "";
in
{
services."backup-${name}" = {
description = "Backup ${title} with Borg";
@ -79,6 +96,7 @@ let
script = ''
${borgCommonSettings}
${logPrefix} # Add verbose logging if enabled
LOG_FILE="/tmp/borg-${name}-backup-$(date +%Y%m%d-%H%M%S).log"
${initRepo repo}
@ -87,13 +105,15 @@ let
ARCHIVE_NAME="${name}-$(date +%Y-%m-%d_%H%M%S)"
START_TIME=$(date +%s)
# Add verbose output redirection if enabled
${if verbose then "exec 3>&1 4>&2" else ""}
${pkgs.borgbackup}/bin/borg create \
--stats \
--compression zstd,15 \
--exclude '*.tmp' \
--exclude '*/tmp/*' \
${repo}::$ARCHIVE_NAME \
${sourcePath} >> $LOG_FILE 2>&1
${sourcePath} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""}
BACKUP_STATUS=$?
END_TIME=$(date +%s)
@ -107,34 +127,33 @@ let
--keep-daily ${toString keepDaily} \
--keep-weekly ${toString keepWeekly} \
--keep-monthly ${toString keepMonthly} \
${repo} >> $LOG_FILE 2>&1
${repo} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""}
PRUNE_STATUS=$?
echo -e "\nRemaining archives after pruning:" >> $LOG_FILE
${pkgs.borgbackup}/bin/borg list ${repo} >> $LOG_FILE 2>&1 || true
if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then
${sendNotification "✅ ${title} Backup Complete" "${title} backup completed successfully on $(hostname) at $(date)\nDuration: $(date -d@$DURATION -u +%H:%M:%S)\n\n$STATS"}
else
${sendNotification "❌ ${title} Backup Failed" "${title} backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nPartial Stats:\n$STATS\n\nSee $LOG_FILE for details"}
fi
${
if enableNotifications then
''
if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then
${sendNotification "✅ ${title} Backup Complete" "${title} backup completed successfully on $(hostname) at $(date)\nDuration: $(date -d@$DURATION -u +%H:%M:%S)\n\n$STATS"}
else
${sendNotification "❌ ${title} Backup Failed" "${title} backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nPartial Stats:\n$STATS\n\nSee $LOG_FILE for details"}
fi
''
else
"echo 'Notifications disabled' >> $LOG_FILE"
}
rm -f $LOG_FILE.stats
exit $BACKUP_STATUS
'';
};
timers."backup-${name}" = {
description = "Timer for ${title} Backup";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = schedule;
Persistent = true;
RandomizedDelaySec = "5min";
};
};
};
}
// maybeCreateTimer;
# Common service configuration
commonServiceConfig = {
@ -151,7 +170,6 @@ in
{
environment.systemPackages = with pkgs; [
borgbackup
apprise
];
systemd = lib.mkMerge [
@ -167,7 +185,10 @@ in
keepDaily = 7;
keepWeekly = 4;
keepMonthly = 3;
schedule = "*-*-* 03:00:00"; # Daily at 3am
# No schedule = no timer created
# schedule = "*-*-* 03:00:00";
enableNotifications = false;
verbose = true;
})
(mkBorgBackupService {
@ -178,7 +199,9 @@ in
keepDaily = 7;
keepWeekly = 4;
keepMonthly = 3;
schedule = "*-*-* 03:00:00";
# schedule = "*-*-* 03:00:00";
enableNotifications = false;
verbose = true;
})
];
}

197
hosts/nixos/cloud/config/backups/default.nix Normal file
View file

@ -0,0 +1,197 @@
{
config,
lib,
pkgs,
...
}:
let
# Shared configuration
logDir = "/var/log/backups";
backupServices = [
{
name = "forgejo";
title = "Forgejo";
service = "backup-forgejo.service";
logPattern = "borg-forgejo-backup-*.log";
}
{
name = "docker_storage";
title = "Docker Storage";
service = "backup-docker-storage.service";
logPattern = "borg-docker-storage-backup-*.log";
}
{
name = "snapraid";
title = "SnapRAID";
service = "snapraid-aio.service";
logPattern = "SnapRAID-*.out";
logPath = "/var/log/snapraid";
}
];
# Helper functions
users = config.secretsSpec.users;
notify =
title: message: logFile:
let
attachArg = if logFile == "" then "" else "--attach \"file://${logFile}\"";
appriseUrl = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo";
in
''
${pkgs.apprise}/bin/apprise -vv -i "markdown" -t "${title}" \
-b "${message}" \
${attachArg} \
"${appriseUrl}" || true
'';
findLatestLog = pattern: path: ''
find "${path}" -name "${pattern}" -type f -printf "%T@ %p\\n" 2>/dev/null \
| sort -nr | head -1 | cut -d' ' -f2
'';
# Generate safe variable name (replace hyphens with underscores)
safeName = name: lib.replaceStrings [ "-" ] [ "_" ] name;
# Generate status variable references
statusVarName = name: "STATUS_${safeName name}";
# Common script utilities
scriptPrelude = ''
set -uo pipefail
LOG_FILE="${logDir}/backup-chain-$(date +%Y%m%d-%H%M%S).log"
mkdir -p "${logDir}"
exec > >(tee -a "$LOG_FILE") 2>&1
log() {
echo "[$(date "+%Y-%m-%d %H:%M:%S")] $1"
}
# Initialize all status variables
${lib.concatMapStringsSep "\n" (s: "${statusVarName s.name}=1") backupServices}
'';
# Service runner template
runService =
{
name,
title,
service,
logPattern,
logPath ? "/tmp",
}:
''
log "Starting ${title} maintenance..."
systemctl start ${service} || true
${statusVarName name}=$?
log "${title} completed with status $${statusVarName name}"
SERVICE_LOG=$(${findLatestLog logPattern logPath})
if [ -n "$SERVICE_LOG" ]; then
log "Appending ${title} log: $SERVICE_LOG"
echo -e "\n\n===== ${title} LOG ($(basename "$SERVICE_LOG")) =====\n" >> "$LOG_FILE"
cat "$SERVICE_LOG" >> "$LOG_FILE"
# Add SnapRAID-specific summary
if [ "${name}" = "snapraid" ]; then
echo -e "\n=== SnapRAID Summary ===" >> "$LOG_FILE"
grep -E '(Scrub|Sync|Diff|smart)' "$SERVICE_LOG" | tail -n 10 >> "$LOG_FILE"
fi
fi
'';
# Build the service execution script
serviceExecution = lib.concatMapStrings runService backupServices;
# Generate status summary lines
statusSummaryLines = lib.concatMapStringsSep "\n" (
s:
let
varName = statusVarName s.name;
in
"- **${s.title}:** \$([ \$${varName} -eq 0 ] && echo '✅ Success' || echo '❌ Failed') (Exit: \$${varName})"
) backupServices;
# Notification logic with cleaner formatting
notificationLogic =
let
statusVars = map (s: statusVarName s.name) backupServices;
statusChecks = lib.concatMapStringsSep "\n" (var: "[ \$${var} -eq 0 ] && ") statusVars;
in
''
# Calculate overall status
OVERALL_STATUS=0
${lib.concatMapStringsSep "\n" (var: "if [ \$${var} -ne 0 ]; then OVERALL_STATUS=1; fi") statusVars}
TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
HOSTNAME=$(hostname)
SUMMARY=$(cat << EOF
# Backup Chain Complete
**Host:** $HOSTNAME
**Timestamp:** $TIMESTAMP
**Overall Status:** $([ $OVERALL_STATUS -eq 0 ] && echo '✅ Success' || echo '⚠️ Failure')
## Service Status:
${statusSummaryLines}
**Log Path:** $LOG_FILE
EOF)
if [ $OVERALL_STATUS -eq 0 ]; then
${notify "✅ Backup Success" "$SUMMARY" "$LOG_FILE"}
else
${notify "⚠️ Backup Issues" "$SUMMARY" "$LOG_FILE"}
fi
exit $OVERALL_STATUS
'';
in
{
imports = lib.custom.scanPaths ./.;
systemd.services.backup-chain = {
description = "Orchestrated Backup Chain";
path = with pkgs; [
apprise
coreutils
findutils
gawk
gnugrep
hostname
systemd
util-linux
];
serviceConfig = {
Type = "oneshot";
Nice = 19;
IOSchedulingClass = "idle";
CPUSchedulingPolicy = "idle";
};
script = ''
${scriptPrelude}
log "Initializing backup chain on $(hostname)"
${serviceExecution}
log "Finalizing backup chain"
${notificationLogic}
'';
};
systemd.timers.backup-chain = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "*-*-* 03:00:00";
Persistent = true;
RandomizedDelaySec = "5min";
};
};
environment.systemPackages = [ pkgs.apprise ];
systemd.tmpfiles.rules = [ "d ${logDir} 0755 root root -" ];
}

15
hosts/nixos/cloud/config/snapraid.nix → hosts/nixos/cloud/config/backups/snapraid.nix
View file

@ -1,12 +1,13 @@
{
pkgs,
inputs,
lib,
config,
...
}:
let
apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl;
users = config.secretsSpec.users;
apprise-url = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo";
snapraid-aio = inputs.snapraid-aio.nixosModules.default;
snapraid-aio-config = pkgs.writeTextFile {
@ -20,7 +21,7 @@ let
APPRISE_URL=""
APPRISE_ATTACH=1
APPRISE_BIN="${pkgs.apprise}/bin/apprise"
APPRISE_EMAIL=1
APPRISE_EMAIL=0
APPRISE_EMAIL_URL="${apprise-url}"
TELEGRAM=0
DISCORD=0
@ -116,19 +117,19 @@ in
inputs.snapraid-aio.nixosModules.default
];
# Make sure the SnapRAID config exists
environment.etc."snapraid.conf".source = snapraid-conf;
# Create required directories
systemd.tmpfiles.rules = [
"d /var/lib/snapraid-aio 0755 root root -"
"d /var/log/snapraid 0755 root root -"
];
environment.systemPackages = [ pkgs.snapraid ];
environment.etc."snapraid.conf".source = snapraid-conf;
# Set up snapraid-aio service
services.snapraid-aio = {
enable = true;
configFile = snapraid-aio-config;
schedule = "*-*-* 04:00:00"; # Run daily at 3am
# schedule = "*-*-* 04:00:00"; # Run daily at 3am
};
}

3
hosts/nixos/cloud/default.nix
View file

@ -60,10 +60,7 @@ in
## System-wide packages ##
programs.nix-ld.enable = true;
environment.systemPackages = with pkgs; [
apprise
lazydocker
mergerfs
snapraid
];
# https://wiki.nixos.org/wiki/FAQ/When_do_I_update_stateVersion

16
lib/default.nix
View file

@ -18,4 +18,20 @@
) (builtins.readDir path)
)
);
# Generate an Apprise URL for sending notifications
# Can be called with smtp config and recipient:
# mkAppriseUrl smtpConfig recipient
# Or with individual parameters:
# mkAppriseUrl { user = "user"; password = "pass"; host = "smtp.example.com"; from = "sender@example.com"; } "recipient@example.com"
mkAppriseUrl =
smtp: recipient:
let
smtpUser = if builtins.isAttrs smtp then smtp.user else smtp;
smtpPass = if builtins.isAttrs smtp then smtp.password else recipient;
smtpHost = if builtins.isAttrs smtp then smtp.host else "";
smtpFrom = if builtins.isAttrs smtp then smtp.from else "";
to = if builtins.isAttrs smtp then recipient else smtp.user;
in
"mailtos://_?user=${smtpUser}&pass=${smtpPass}&smtp=${smtpHost}&from=${smtpFrom}&to=${to}";
}

18
modules/common/secret-spec.nix
View file

@ -18,17 +18,6 @@ let
grep -q "BEGIN OPENSSH PRIVATE KEY" "$out" || (echo "Invalid SSH key format"; exit 1)
'';
};
# Function to build an Apprise URL from SMTP settings
buildAppriseUrl =
{
host,
user,
password,
from,
...
}:
"mailtos://_?user=${user}&pass=${password}&smtp=${host}&from=${from}&to=${user}";
in
{
options.secretsSpec = {
@ -126,13 +115,6 @@ in
type = lib.types.str;
description = "Email address to send from";
};
notifyUrl = lib.mkOption {
type = lib.types.str;
description = "Apprise URL for sending notifications via this SMTP account";
};
};
config = {
notifyUrl = "mailtos://_?user=${config.user}&pass=${config.password}&smtp=${config.host}&from=${config.from}&to=${config.user}";
};
}
);