Enhances backup orchestration and notifications

• Move lazydocker pkg to docker.nix
• Extends backup service generator with optional scheduling, verbose logging, and new mkAppriseUrl
• Refactors backup configurations and renames files for clarity
• Introduces backup chain orchestration for smoother maintenance
• Updates Apprise URL generation and removes deprecated secret spec functions

hosts/common/optional/docker.nix

@@ -7,4 +7,8 @@
     };
     oci-containers.backend = "docker";
   };
+
+  environment.systemPackages = with pkgs; [
+    lazydocker
+  ];
 }

hosts/nixos/cloud/config/backups/borg.nix

@@ -60,7 +60,7 @@ let
     STATS=$(cat ${logFile}.stats || echo "No stats available")
   '';
 
-  # Unified backup service generator
+  # Unified backup service generator with optional features
   mkBorgBackupService =
     {
       name,
@@ -70,8 +70,25 @@ let
       keepDaily,
       keepWeekly,
       keepMonthly,
-      schedule,
+      schedule ? null,
+      enableNotifications ? true,
+      verbose ? false,
     }:
+    let
+      maybeCreateTimer = lib.optionalAttrs (schedule != null) {
+        timers."backup-${name}" = {
+          description = "Timer for ${title} Backup";
+          wantedBy = [ "timers.target" ];
+          timerConfig = {
+            OnCalendar = schedule;
+            Persistent = true;
+            RandomizedDelaySec = "5min";
+          };
+        };
+      };
+
+      logPrefix = if verbose then "set -x;" else "";
+    in
     {
       services."backup-${name}" = {
         description = "Backup ${title} with Borg";
@@ -79,6 +96,7 @@ let
 
         script = ''
           ${borgCommonSettings}
+          ${logPrefix}  # Add verbose logging if enabled
 
           LOG_FILE="/tmp/borg-${name}-backup-$(date +%Y%m%d-%H%M%S).log"
           ${initRepo repo}
@@ -87,13 +105,15 @@ let
           ARCHIVE_NAME="${name}-$(date +%Y-%m-%d_%H%M%S)"
           START_TIME=$(date +%s)
 
+          # Add verbose output redirection if enabled
+          ${if verbose then "exec 3>&1 4>&2" else ""}
           ${pkgs.borgbackup}/bin/borg create \
             --stats \
             --compression zstd,15 \
             --exclude '*.tmp' \
             --exclude '*/tmp/*' \
             ${repo}::$ARCHIVE_NAME \
-            ${sourcePath} >> $LOG_FILE 2>&1
+            ${sourcePath} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""}
 
           BACKUP_STATUS=$?
           END_TIME=$(date +%s)
@@ -107,34 +127,33 @@ let
             --keep-daily ${toString keepDaily} \
             --keep-weekly ${toString keepWeekly} \
             --keep-monthly ${toString keepMonthly} \
-            ${repo} >> $LOG_FILE 2>&1
+            ${repo} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""}
 
           PRUNE_STATUS=$?
 
           echo -e "\nRemaining archives after pruning:" >> $LOG_FILE
           ${pkgs.borgbackup}/bin/borg list ${repo} >> $LOG_FILE 2>&1 || true
 
-          if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then
+          ${
-            ${sendNotification "✅ ${title} Backup Complete" "${title} backup completed successfully on $(hostname) at $(date)\nDuration: $(date -d@$DURATION -u +%H:%M:%S)\n\n$STATS"}
+            if enableNotifications then
-          else
+              ''
-            ${sendNotification "❌ ${title} Backup Failed" "${title} backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nPartial Stats:\n$STATS\n\nSee $LOG_FILE for details"}
+                if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then
-          fi
+                  ${sendNotification "✅ ${title} Backup Complete" "${title} backup completed successfully on $(hostname) at $(date)\nDuration: $(date -d@$DURATION -u +%H:%M:%S)\n\n$STATS"}
+                else
+                  ${sendNotification "❌ ${title} Backup Failed" "${title} backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nPartial Stats:\n$STATS\n\nSee $LOG_FILE for details"}
+                fi
+              ''
+            else
+              "echo 'Notifications disabled' >> $LOG_FILE"
+          }
 
           rm -f $LOG_FILE.stats
           exit $BACKUP_STATUS
         '';
       };
 
-      timers."backup-${name}" = {
+    }
-        description = "Timer for ${title} Backup";
+    // maybeCreateTimer;
-        wantedBy = [ "timers.target" ];
-        timerConfig = {
-          OnCalendar = schedule;
-          Persistent = true;
-          RandomizedDelaySec = "5min";
-        };
-      };
-    };
 
   # Common service configuration
   commonServiceConfig = {
@@ -151,7 +170,6 @@ in
 {
   environment.systemPackages = with pkgs; [
     borgbackup
-    apprise
   ];
 
   systemd = lib.mkMerge [
@@ -160,10 +178,17 @@ in
       title = "Docker Storage";
       repo = dockerStorageRepo;
       sourcePath = "/mnt/drive1/DockerStorage";
+      # INFO: This shit confusing but basically
+      # keeps the last 7 days,
+      # then keeps AT LEAST ONE for last 4 weeks
+      # and finally AT LEAST ONE for the last 3 months
       keepDaily = 7;
       keepWeekly = 4;
       keepMonthly = 3;
-      schedule = "Mon *-*-* 04:00:00";
+      # No schedule = no timer created
+      # schedule = "*-*-* 03:00:00";
+      enableNotifications = false;
+      verbose = true;
     })
 
     (mkBorgBackupService {
@@ -171,10 +196,12 @@ in
       title = "Forgejo";
       repo = forgejoRepo;
       sourcePath = "/pool/forgejo";
-      keepDaily = 14;
+      keepDaily = 7;
       keepWeekly = 4;
       keepMonthly = 3;
-      schedule = "*-*-1/2 04:00:00";
+      # schedule = "*-*-* 03:00:00";
+      enableNotifications = false;
+      verbose = true;
     })
   ];
 }

hosts/nixos/cloud/config/backups/default.nix

@@ -0,0 +1,197 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  # Shared configuration
+  logDir = "/var/log/backups";
+  backupServices = [
+    {
+      name = "forgejo";
+      title = "Forgejo";
+      service = "backup-forgejo.service";
+      logPattern = "borg-forgejo-backup-*.log";
+    }
+    {
+      name = "docker_storage";
+      title = "Docker Storage";
+      service = "backup-docker-storage.service";
+      logPattern = "borg-docker-storage-backup-*.log";
+    }
+    {
+      name = "snapraid";
+      title = "SnapRAID";
+      service = "snapraid-aio.service";
+      logPattern = "SnapRAID-*.out";
+      logPath = "/var/log/snapraid";
+    }
+  ];
+
+  # Helper functions
+  users = config.secretsSpec.users;
+  notify =
+    title: message: logFile:
+    let
+      attachArg = if logFile == "" then "" else "--attach \"file://${logFile}\"";
+      appriseUrl = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo";
+    in
+    ''
+      ${pkgs.apprise}/bin/apprise  -vv -i "markdown" -t "${title}" \
+        -b "${message}" \
+        ${attachArg} \
+        "${appriseUrl}" || true
+    '';
+
+  findLatestLog = pattern: path: ''
+    find "${path}" -name "${pattern}" -type f -printf "%T@ %p\\n" 2>/dev/null \
+      | sort -nr | head -1 | cut -d' ' -f2
+  '';
+
+  # Generate safe variable name (replace hyphens with underscores)
+  safeName = name: lib.replaceStrings [ "-" ] [ "_" ] name;
+
+  # Generate status variable references
+  statusVarName = name: "STATUS_${safeName name}";
+
+  # Common script utilities
+  scriptPrelude = ''
+    set -uo pipefail
+    LOG_FILE="${logDir}/backup-chain-$(date +%Y%m%d-%H%M%S).log"
+    mkdir -p "${logDir}"
+    exec > >(tee -a "$LOG_FILE") 2>&1
+
+    log() {
+      echo "[$(date "+%Y-%m-%d %H:%M:%S")] $1"
+    }
+
+    # Initialize all status variables
+    ${lib.concatMapStringsSep "\n" (s: "${statusVarName s.name}=1") backupServices}
+  '';
+
+  # Service runner template
+  runService =
+    {
+      name,
+      title,
+      service,
+      logPattern,
+      logPath ? "/tmp",
+    }:
+    ''
+      log "Starting ${title} maintenance..."
+      systemctl start ${service} || true
+      ${statusVarName name}=$?
+      log "${title} completed with status $${statusVarName name}"
+
+      SERVICE_LOG=$(${findLatestLog logPattern logPath})
+      if [ -n "$SERVICE_LOG" ]; then
+        log "Appending ${title} log: $SERVICE_LOG"
+        echo -e "\n\n===== ${title} LOG ($(basename "$SERVICE_LOG")) =====\n" >> "$LOG_FILE"
+        cat "$SERVICE_LOG" >> "$LOG_FILE"
+        
+        # Add SnapRAID-specific summary
+        if [ "${name}" = "snapraid" ]; then
+          echo -e "\n=== SnapRAID Summary ===" >> "$LOG_FILE"
+          grep -E '(Scrub|Sync|Diff|smart)' "$SERVICE_LOG" | tail -n 10 >> "$LOG_FILE"
+        fi
+      fi
+    '';
+
+  # Build the service execution script
+  serviceExecution = lib.concatMapStrings runService backupServices;
+
+  # Generate status summary lines
+  statusSummaryLines = lib.concatMapStringsSep "\n" (
+    s:
+    let
+      varName = statusVarName s.name;
+    in
+    "- **${s.title}:** \$([ \$${varName} -eq 0 ] && echo '✅ Success' || echo '❌ Failed') (Exit: \$${varName})"
+  ) backupServices;
+
+  # Notification logic with cleaner formatting
+  notificationLogic =
+    let
+      statusVars = map (s: statusVarName s.name) backupServices;
+      statusChecks = lib.concatMapStringsSep "\n" (var: "[ \$${var} -eq 0 ] && ") statusVars;
+    in
+    ''
+      # Calculate overall status
+      OVERALL_STATUS=0
+      ${lib.concatMapStringsSep "\n" (var: "if [ \$${var} -ne 0 ]; then OVERALL_STATUS=1; fi") statusVars}
+
+      TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
+      HOSTNAME=$(hostname)
+
+      SUMMARY=$(cat << EOF
+      # Backup Chain Complete
+
+      **Host:** $HOSTNAME  
+      **Timestamp:** $TIMESTAMP  
+      **Overall Status:** $([ $OVERALL_STATUS -eq 0 ] && echo '✅ Success' || echo '⚠️ Failure')
+
+      ## Service Status:
+      ${statusSummaryLines}
+
+      **Log Path:** $LOG_FILE
+      EOF)
+
+      if [ $OVERALL_STATUS -eq 0 ]; then
+        ${notify "✅ Backup Success" "$SUMMARY" "$LOG_FILE"}
+      else
+        ${notify "⚠️ Backup Issues" "$SUMMARY" "$LOG_FILE"}
+      fi
+
+      exit $OVERALL_STATUS
+    '';
+
+in
+{
+  imports = lib.custom.scanPaths ./.;
+
+  systemd.services.backup-chain = {
+    description = "Orchestrated Backup Chain";
+    path = with pkgs; [
+      apprise
+      coreutils
+      findutils
+      gawk
+      gnugrep
+      hostname
+      systemd
+      util-linux
+    ];
+
+    serviceConfig = {
+      Type = "oneshot";
+      Nice = 19;
+      IOSchedulingClass = "idle";
+      CPUSchedulingPolicy = "idle";
+    };
+
+    script = ''
+      ${scriptPrelude}
+      log "Initializing backup chain on $(hostname)"
+
+      ${serviceExecution}
+
+      log "Finalizing backup chain"
+      ${notificationLogic}
+    '';
+  };
+
+  systemd.timers.backup-chain = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*-*-* 03:00:00";
+      Persistent = true;
+      RandomizedDelaySec = "5min";
+    };
+  };
+
+  environment.systemPackages = [ pkgs.apprise ];
+  systemd.tmpfiles.rules = [ "d ${logDir} 0755 root root -" ];
+}

hosts/nixos/cloud/config/backups/snapraid.nix

@@ -1,12 +1,13 @@
 {
   pkgs,
   inputs,
+  lib,
   config,
   ...
 }:
 let
-
+  users = config.secretsSpec.users;
-  apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl;
+  apprise-url = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo";
 
   snapraid-aio = inputs.snapraid-aio.nixosModules.default;
   snapraid-aio-config = pkgs.writeTextFile {
@@ -20,7 +21,7 @@ let
       APPRISE_URL=""
       APPRISE_ATTACH=1
       APPRISE_BIN="${pkgs.apprise}/bin/apprise"
-      APPRISE_EMAIL=1
+      APPRISE_EMAIL=0
       APPRISE_EMAIL_URL="${apprise-url}"
       TELEGRAM=0
       DISCORD=0
@@ -104,10 +105,10 @@ let
       exclude .Trash/
       # These dirs change data all the time
       # so I back them up in borg repos that are not excluded 
-      exclude /mnt/drive1/DockerStorage/
+      exclude /DockerStorage/
-      exclude /mnt/drive1/data/forgejo
+      exclude /data/forgejo/
-      exclude /mnt/drive2/data/forgejo
+      exclude /data/forgejo/
-      exclude /mnt/drive3/data/forgejo
+      exclude /data/forgejo/
     '';
   };
 in
@@ -116,19 +117,19 @@ in
     inputs.snapraid-aio.nixosModules.default
   ];
 
-  # Make sure the SnapRAID config exists
-  environment.etc."snapraid.conf".source = snapraid-conf;
-
   # Create required directories
   systemd.tmpfiles.rules = [
     "d /var/lib/snapraid-aio 0755 root root -"
     "d /var/log/snapraid 0755 root root -"
   ];
 
+  environment.systemPackages = [ pkgs.snapraid ];
+  environment.etc."snapraid.conf".source = snapraid-conf;
+
   # Set up snapraid-aio service
   services.snapraid-aio = {
     enable = true;
     configFile = snapraid-aio-config;
-    schedule = "*-*-* 03:00:00"; # Run daily at 3am
+    # schedule = "*-*-* 04:00:00"; # Run daily at 3am
   };
 }

hosts/nixos/cloud/default.nix

@@ -60,10 +60,7 @@ in
   ## System-wide packages ##
   programs.nix-ld.enable = true;
   environment.systemPackages = with pkgs; [
-    apprise
-    lazydocker
     mergerfs
-    snapraid
   ];
 
   # https://wiki.nixos.org/wiki/FAQ/When_do_I_update_stateVersion

lib/default.nix

@@ -18,4 +18,20 @@
         ) (builtins.readDir path)
       )
     );
+
+  # Generate an Apprise URL for sending notifications
+  # Can be called with smtp config and recipient:
+  # mkAppriseUrl smtpConfig recipient
+  # Or with individual parameters:
+  # mkAppriseUrl { user = "user"; password = "pass"; host = "smtp.example.com"; from = "sender@example.com"; } "recipient@example.com"
+  mkAppriseUrl =
+    smtp: recipient:
+    let
+      smtpUser = if builtins.isAttrs smtp then smtp.user else smtp;
+      smtpPass = if builtins.isAttrs smtp then smtp.password else recipient;
+      smtpHost = if builtins.isAttrs smtp then smtp.host else "";
+      smtpFrom = if builtins.isAttrs smtp then smtp.from else "";
+      to = if builtins.isAttrs smtp then recipient else smtp.user;
+    in
+    "mailtos://_?user=${smtpUser}&pass=${smtpPass}&smtp=${smtpHost}&from=${smtpFrom}&to=${to}";
 }

modules/common/secret-spec.nix

@@ -18,17 +18,6 @@ let
         grep -q "BEGIN OPENSSH PRIVATE KEY" "$out" || (echo "Invalid SSH key format"; exit 1)
       '';
     };
-
-  # Function to build an Apprise URL from SMTP settings
-  buildAppriseUrl =
-    {
-      host,
-      user,
-      password,
-      from,
-      ...
-    }:
-    "mailtos://_?user=${user}&pass=${password}&smtp=${host}&from=${from}&to=${user}";
 in
 {
   options.secretsSpec = {
@@ -126,13 +115,6 @@ in
                       type = lib.types.str;
                       description = "Email address to send from";
                     };
-                    notifyUrl = lib.mkOption {
-                      type = lib.types.str;
-                      description = "Apprise URL for sending notifications via this SMTP account";
-                    };
-                  };
-                  config = {
-                    notifyUrl = "mailtos://_?user=${config.user}&pass=${config.password}&smtp=${config.host}&from=${config.from}&to=${config.user}";
                   };
                 }
               );