diff --git a/hosts/common/optional/docker.nix b/hosts/common/optional/docker.nix index 33cf96f..2a10967 100644 --- a/hosts/common/optional/docker.nix +++ b/hosts/common/optional/docker.nix @@ -7,4 +7,8 @@ }; oci-containers.backend = "docker"; }; + + environment.systemPackages = with pkgs; [ + lazydocker + ]; } diff --git a/hosts/nixos/cloud/config/borg.nix b/hosts/nixos/cloud/config/backups/borg.nix similarity index 70% rename from hosts/nixos/cloud/config/borg.nix rename to hosts/nixos/cloud/config/backups/borg.nix index 443d37b..fdec1c6 100644 --- a/hosts/nixos/cloud/config/borg.nix +++ b/hosts/nixos/cloud/config/backups/borg.nix @@ -60,7 +60,7 @@ let STATS=$(cat ${logFile}.stats || echo "No stats available") ''; - # Unified backup service generator + # Unified backup service generator with optional features mkBorgBackupService = { name, @@ -70,8 +70,25 @@ let keepDaily, keepWeekly, keepMonthly, - schedule, + schedule ? null, + enableNotifications ? true, + verbose ? false, }: + let + maybeCreateTimer = lib.optionalAttrs (schedule != null) { + timers."backup-${name}" = { + description = "Timer for ${title} Backup"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = schedule; + Persistent = true; + RandomizedDelaySec = "5min"; + }; + }; + }; + + logPrefix = if verbose then "set -x;" else ""; + in { services."backup-${name}" = { description = "Backup ${title} with Borg"; @@ -79,6 +96,7 @@ let script = '' ${borgCommonSettings} + ${logPrefix} # Add verbose logging if enabled LOG_FILE="/tmp/borg-${name}-backup-$(date +%Y%m%d-%H%M%S).log" ${initRepo repo} @@ -87,13 +105,15 @@ let ARCHIVE_NAME="${name}-$(date +%Y-%m-%d_%H%M%S)" START_TIME=$(date +%s) + # Add verbose output redirection if enabled + ${if verbose then "exec 3>&1 4>&2" else ""} ${pkgs.borgbackup}/bin/borg create \ --stats \ --compression zstd,15 \ --exclude '*.tmp' \ --exclude '*/tmp/*' \ ${repo}::$ARCHIVE_NAME \ - ${sourcePath} >> $LOG_FILE 2>&1 + ${sourcePath} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""} BACKUP_STATUS=$? END_TIME=$(date +%s) @@ -107,34 +127,33 @@ let --keep-daily ${toString keepDaily} \ --keep-weekly ${toString keepWeekly} \ --keep-monthly ${toString keepMonthly} \ - ${repo} >> $LOG_FILE 2>&1 + ${repo} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""} PRUNE_STATUS=$? echo -e "\nRemaining archives after pruning:" >> $LOG_FILE ${pkgs.borgbackup}/bin/borg list ${repo} >> $LOG_FILE 2>&1 || true - if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then - ${sendNotification "✅ ${title} Backup Complete" "${title} backup completed successfully on $(hostname) at $(date)\nDuration: $(date -d@$DURATION -u +%H:%M:%S)\n\n$STATS"} - else - ${sendNotification "❌ ${title} Backup Failed" "${title} backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nPartial Stats:\n$STATS\n\nSee $LOG_FILE for details"} - fi + ${ + if enableNotifications then + '' + if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then + ${sendNotification "✅ ${title} Backup Complete" "${title} backup completed successfully on $(hostname) at $(date)\nDuration: $(date -d@$DURATION -u +%H:%M:%S)\n\n$STATS"} + else + ${sendNotification "❌ ${title} Backup Failed" "${title} backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nPartial Stats:\n$STATS\n\nSee $LOG_FILE for details"} + fi + '' + else + "echo 'Notifications disabled' >> $LOG_FILE" + } rm -f $LOG_FILE.stats exit $BACKUP_STATUS ''; }; - timers."backup-${name}" = { - description = "Timer for ${title} Backup"; - wantedBy = [ "timers.target" ]; - timerConfig = { - OnCalendar = schedule; - Persistent = true; - RandomizedDelaySec = "5min"; - }; - }; - }; + } + // maybeCreateTimer; # Common service configuration commonServiceConfig = { @@ -151,7 +170,6 @@ in { environment.systemPackages = with pkgs; [ borgbackup - apprise ]; systemd = lib.mkMerge [ @@ -167,7 +185,10 @@ in keepDaily = 7; keepWeekly = 4; keepMonthly = 3; - schedule = "*-*-* 03:00:00"; # Daily at 3am + # No schedule = no timer created + # schedule = "*-*-* 03:00:00"; + enableNotifications = false; + verbose = true; }) (mkBorgBackupService { @@ -178,7 +199,9 @@ in keepDaily = 7; keepWeekly = 4; keepMonthly = 3; - schedule = "*-*-* 03:00:00"; + # schedule = "*-*-* 03:00:00"; + enableNotifications = false; + verbose = true; }) ]; } diff --git a/hosts/nixos/cloud/config/backups/default.nix b/hosts/nixos/cloud/config/backups/default.nix new file mode 100644 index 0000000..7d77f85 --- /dev/null +++ b/hosts/nixos/cloud/config/backups/default.nix @@ -0,0 +1,197 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # Shared configuration + logDir = "/var/log/backups"; + backupServices = [ + { + name = "forgejo"; + title = "Forgejo"; + service = "backup-forgejo.service"; + logPattern = "borg-forgejo-backup-*.log"; + } + { + name = "docker_storage"; + title = "Docker Storage"; + service = "backup-docker-storage.service"; + logPattern = "borg-docker-storage-backup-*.log"; + } + { + name = "snapraid"; + title = "SnapRAID"; + service = "snapraid-aio.service"; + logPattern = "SnapRAID-*.out"; + logPath = "/var/log/snapraid"; + } + ]; + + # Helper functions + users = config.secretsSpec.users; + notify = + title: message: logFile: + let + attachArg = if logFile == "" then "" else "--attach \"file://${logFile}\""; + appriseUrl = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo"; + in + '' + ${pkgs.apprise}/bin/apprise -vv -i "markdown" -t "${title}" \ + -b "${message}" \ + ${attachArg} \ + "${appriseUrl}" || true + ''; + + findLatestLog = pattern: path: '' + find "${path}" -name "${pattern}" -type f -printf "%T@ %p\\n" 2>/dev/null \ + | sort -nr | head -1 | cut -d' ' -f2 + ''; + + # Generate safe variable name (replace hyphens with underscores) + safeName = name: lib.replaceStrings [ "-" ] [ "_" ] name; + + # Generate status variable references + statusVarName = name: "STATUS_${safeName name}"; + + # Common script utilities + scriptPrelude = '' + set -uo pipefail + LOG_FILE="${logDir}/backup-chain-$(date +%Y%m%d-%H%M%S).log" + mkdir -p "${logDir}" + exec > >(tee -a "$LOG_FILE") 2>&1 + + log() { + echo "[$(date "+%Y-%m-%d %H:%M:%S")] $1" + } + + # Initialize all status variables + ${lib.concatMapStringsSep "\n" (s: "${statusVarName s.name}=1") backupServices} + ''; + + # Service runner template + runService = + { + name, + title, + service, + logPattern, + logPath ? "/tmp", + }: + '' + log "Starting ${title} maintenance..." + systemctl start ${service} || true + ${statusVarName name}=$? + log "${title} completed with status $${statusVarName name}" + + SERVICE_LOG=$(${findLatestLog logPattern logPath}) + if [ -n "$SERVICE_LOG" ]; then + log "Appending ${title} log: $SERVICE_LOG" + echo -e "\n\n===== ${title} LOG ($(basename "$SERVICE_LOG")) =====\n" >> "$LOG_FILE" + cat "$SERVICE_LOG" >> "$LOG_FILE" + + # Add SnapRAID-specific summary + if [ "${name}" = "snapraid" ]; then + echo -e "\n=== SnapRAID Summary ===" >> "$LOG_FILE" + grep -E '(Scrub|Sync|Diff|smart)' "$SERVICE_LOG" | tail -n 10 >> "$LOG_FILE" + fi + fi + ''; + + # Build the service execution script + serviceExecution = lib.concatMapStrings runService backupServices; + + # Generate status summary lines + statusSummaryLines = lib.concatMapStringsSep "\n" ( + s: + let + varName = statusVarName s.name; + in + "- **${s.title}:** \$([ \$${varName} -eq 0 ] && echo '✅ Success' || echo '❌ Failed') (Exit: \$${varName})" + ) backupServices; + + # Notification logic with cleaner formatting + notificationLogic = + let + statusVars = map (s: statusVarName s.name) backupServices; + statusChecks = lib.concatMapStringsSep "\n" (var: "[ \$${var} -eq 0 ] && ") statusVars; + in + '' + # Calculate overall status + OVERALL_STATUS=0 + ${lib.concatMapStringsSep "\n" (var: "if [ \$${var} -ne 0 ]; then OVERALL_STATUS=1; fi") statusVars} + + TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") + HOSTNAME=$(hostname) + + SUMMARY=$(cat << EOF + # Backup Chain Complete + + **Host:** $HOSTNAME + **Timestamp:** $TIMESTAMP + **Overall Status:** $([ $OVERALL_STATUS -eq 0 ] && echo '✅ Success' || echo '⚠️ Failure') + + ## Service Status: + ${statusSummaryLines} + + **Log Path:** $LOG_FILE + EOF) + + if [ $OVERALL_STATUS -eq 0 ]; then + ${notify "✅ Backup Success" "$SUMMARY" "$LOG_FILE"} + else + ${notify "⚠️ Backup Issues" "$SUMMARY" "$LOG_FILE"} + fi + + exit $OVERALL_STATUS + ''; + +in +{ + imports = lib.custom.scanPaths ./.; + + systemd.services.backup-chain = { + description = "Orchestrated Backup Chain"; + path = with pkgs; [ + apprise + coreutils + findutils + gawk + gnugrep + hostname + systemd + util-linux + ]; + + serviceConfig = { + Type = "oneshot"; + Nice = 19; + IOSchedulingClass = "idle"; + CPUSchedulingPolicy = "idle"; + }; + + script = '' + ${scriptPrelude} + log "Initializing backup chain on $(hostname)" + + ${serviceExecution} + + log "Finalizing backup chain" + ${notificationLogic} + ''; + }; + + systemd.timers.backup-chain = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*-*-* 03:00:00"; + Persistent = true; + RandomizedDelaySec = "5min"; + }; + }; + + environment.systemPackages = [ pkgs.apprise ]; + systemd.tmpfiles.rules = [ "d ${logDir} 0755 root root -" ]; +} diff --git a/hosts/nixos/cloud/config/snapraid.nix b/hosts/nixos/cloud/config/backups/snapraid.nix similarity index 92% rename from hosts/nixos/cloud/config/snapraid.nix rename to hosts/nixos/cloud/config/backups/snapraid.nix index 2174f93..6cd0a23 100644 --- a/hosts/nixos/cloud/config/snapraid.nix +++ b/hosts/nixos/cloud/config/backups/snapraid.nix @@ -1,12 +1,13 @@ { pkgs, inputs, + lib, config, ... }: let - - apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl; + users = config.secretsSpec.users; + apprise-url = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo"; snapraid-aio = inputs.snapraid-aio.nixosModules.default; snapraid-aio-config = pkgs.writeTextFile { @@ -20,7 +21,7 @@ let APPRISE_URL="" APPRISE_ATTACH=1 APPRISE_BIN="${pkgs.apprise}/bin/apprise" - APPRISE_EMAIL=1 + APPRISE_EMAIL=0 APPRISE_EMAIL_URL="${apprise-url}" TELEGRAM=0 DISCORD=0 @@ -116,19 +117,19 @@ in inputs.snapraid-aio.nixosModules.default ]; - # Make sure the SnapRAID config exists - environment.etc."snapraid.conf".source = snapraid-conf; - # Create required directories systemd.tmpfiles.rules = [ "d /var/lib/snapraid-aio 0755 root root -" "d /var/log/snapraid 0755 root root -" ]; + environment.systemPackages = [ pkgs.snapraid ]; + environment.etc."snapraid.conf".source = snapraid-conf; + # Set up snapraid-aio service services.snapraid-aio = { enable = true; configFile = snapraid-aio-config; - schedule = "*-*-* 04:00:00"; # Run daily at 3am + # schedule = "*-*-* 04:00:00"; # Run daily at 3am }; } diff --git a/hosts/nixos/cloud/default.nix b/hosts/nixos/cloud/default.nix index 2028e92..4b4055a 100644 --- a/hosts/nixos/cloud/default.nix +++ b/hosts/nixos/cloud/default.nix @@ -60,10 +60,7 @@ in ## System-wide packages ## programs.nix-ld.enable = true; environment.systemPackages = with pkgs; [ - apprise - lazydocker mergerfs - snapraid ]; # https://wiki.nixos.org/wiki/FAQ/When_do_I_update_stateVersion diff --git a/lib/default.nix b/lib/default.nix index b4ec6eb..d33c0b0 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -18,4 +18,20 @@ ) (builtins.readDir path) ) ); + + # Generate an Apprise URL for sending notifications + # Can be called with smtp config and recipient: + # mkAppriseUrl smtpConfig recipient + # Or with individual parameters: + # mkAppriseUrl { user = "user"; password = "pass"; host = "smtp.example.com"; from = "sender@example.com"; } "recipient@example.com" + mkAppriseUrl = + smtp: recipient: + let + smtpUser = if builtins.isAttrs smtp then smtp.user else smtp; + smtpPass = if builtins.isAttrs smtp then smtp.password else recipient; + smtpHost = if builtins.isAttrs smtp then smtp.host else ""; + smtpFrom = if builtins.isAttrs smtp then smtp.from else ""; + to = if builtins.isAttrs smtp then recipient else smtp.user; + in + "mailtos://_?user=${smtpUser}&pass=${smtpPass}&smtp=${smtpHost}&from=${smtpFrom}&to=${to}"; } diff --git a/modules/common/secret-spec.nix b/modules/common/secret-spec.nix index 000f04c..4e2b06f 100644 --- a/modules/common/secret-spec.nix +++ b/modules/common/secret-spec.nix @@ -18,17 +18,6 @@ let grep -q "BEGIN OPENSSH PRIVATE KEY" "$out" || (echo "Invalid SSH key format"; exit 1) ''; }; - - # Function to build an Apprise URL from SMTP settings - buildAppriseUrl = - { - host, - user, - password, - from, - ... - }: - "mailtos://_?user=${user}&pass=${password}&smtp=${host}&from=${from}&to=${user}"; in { options.secretsSpec = { @@ -126,13 +115,6 @@ in type = lib.types.str; description = "Email address to send from"; }; - notifyUrl = lib.mkOption { - type = lib.types.str; - description = "Apprise URL for sending notifications via this SMTP account"; - }; - }; - config = { - notifyUrl = "mailtos://_?user=${config.user}&pass=${config.password}&smtp=${config.host}&from=${config.from}&to=${config.user}"; }; } );