Adds backup notifications & SMTP config

• Incorporates Apprise for sending backup start and completion notifications
• Captures and formats backup statistics for richer log details
• Updates backup exclusion lists to prevent unneeded data inclusion
• Standardizes and extends SMTP configuration in secrets management

flake.lock

@@ -106,7 +106,7 @@
     "crane": {
       "inputs": {
         "flake-compat": "flake-compat_2",
-        "flake-utils": "flake-utils_5",
+        "flake-utils": "flake-utils_6",
         "nixpkgs": [
           "watershot",
           "std",
@@ -308,6 +308,24 @@
       }
     },
     "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_4"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
       "inputs": {
         "systems": [
           "stylix",
@@ -328,9 +346,9 @@
         "type": "github"
       }
     },
-    "flake-utils_3": {
+    "flake-utils_4": {
       "inputs": {
-        "systems": "systems_5"
+        "systems": "systems_6"
       },
       "locked": {
         "lastModified": 1681202837,
@@ -346,7 +364,7 @@
         "type": "github"
       }
     },
-    "flake-utils_4": {
+    "flake-utils_5": {
       "locked": {
         "lastModified": 1659877975,
         "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
@@ -361,7 +379,7 @@
         "type": "github"
       }
     },
-    "flake-utils_5": {
+    "flake-utils_6": {
       "locked": {
         "lastModified": 1667395993,
         "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
@@ -376,6 +394,24 @@
         "type": "github"
       }
     },
+    "flake-utils_7": {
+      "inputs": {
+        "systems": "systems_7"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "fromYaml": {
       "flake": false,
       "locked": {
@@ -940,9 +976,11 @@
         "nixpkgs-unstable": "nixpkgs-unstable",
         "nixvirt": "nixvirt",
         "rose-pine-hyprcursor": "rose-pine-hyprcursor",
+        "snapraid-aio": "snapraid-aio",
         "stylix": "stylix",
         "vscode-server": "vscode-server",
         "watershot": "watershot",
+        "yay": "yay",
         "zen-browser": "zen-browser"
       }
     },
@@ -1014,6 +1052,45 @@
         "type": "github"
       }
     },
+    "snapraid-aio": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "snapraid-aio-src": "snapraid-aio-src"
+      },
+      "locked": {
+        "lastModified": 1745877167,
+        "narHash": "sha256-I1LF6QlQnQmpsom676VNWzbA5xY1ksgwMyPh1b5JoG0=",
+        "ref": "refs/heads/main",
+        "rev": "fb1b3606270ce8ceaea8534a046eb1dda08c54dc",
+        "revCount": 1,
+        "type": "git",
+        "url": "https://git.ryot.foo/toph/snapraid-aio.nix.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.ryot.foo/toph/snapraid-aio.nix.git"
+      }
+    },
+    "snapraid-aio-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1744884143,
+        "narHash": "sha256-GNXn/V4HoFnQtyq7l+V+aXHArObr3zQd4vCgPEqPeRk=",
+        "owner": "auanasgheps",
+        "repo": "snapraid-aio-script",
+        "rev": "a46c7362af385eac945e86a2a0f6097dbe7ca3fb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "auanasgheps",
+        "repo": "snapraid-aio-script",
+        "rev": "a46c7362af385eac945e86a2a0f6097dbe7ca3fb",
+        "type": "github"
+      }
+    },
     "std": {
       "inputs": {
         "arion": [
@@ -1024,7 +1101,7 @@
         "blank": "blank",
         "devshell": "devshell",
         "dmerge": "dmerge",
-        "flake-utils": "flake-utils_4",
+        "flake-utils": "flake-utils_5",
         "incl": "incl",
         "makes": [
           "watershot",
@@ -1069,13 +1146,13 @@
         "base16-vim": "base16-vim",
         "firefox-gnome-theme": "firefox-gnome-theme",
         "flake-compat": "flake-compat",
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils_3",
         "git-hooks": "git-hooks",
         "gnome-shell": "gnome-shell",
         "home-manager": "home-manager_2",
         "nixpkgs": "nixpkgs_3",
         "nur": "nur",
-        "systems": "systems_4",
+        "systems": "systems_5",
         "tinted-foot": "tinted-foot",
         "tinted-kitty": "tinted-kitty",
         "tinted-schemes": "tinted-schemes",
@@ -1171,6 +1248,36 @@
         "type": "github"
       }
     },
+    "systems_6": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_7": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "tinted-foot": {
       "flake": false,
       "locked": {
@@ -1295,7 +1402,7 @@
     },
     "vscode-server": {
       "inputs": {
-        "flake-utils": "flake-utils_3",
+        "flake-utils": "flake-utils_4",
         "nixpkgs": [
           "nixpkgs-unstable"
         ]
@@ -1358,6 +1465,27 @@
         "type": "github"
       }
     },
+    "yay": {
+      "inputs": {
+        "flake-utils": "flake-utils_7",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1745989032,
+        "narHash": "sha256-qKy5YVu8vhA60VxWpLiLV9QpN8LofL9qFCEAACrCxBw=",
+        "ref": "refs/heads/main",
+        "rev": "92d557d0d0393713cb57a970e880efafe6cc2b41",
+        "revCount": 9,
+        "type": "git",
+        "url": "https://git.ryot.foo/toph/yay.nix.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.ryot.foo/toph/yay.nix.git"
+      }
+    },
     "zen-browser": {
       "inputs": {
         "nixpkgs": [

hosts/nixos/cloud/config/borg.nix

@@ -27,10 +27,28 @@ let
     fi
   '';
 
+  # Function to send notifications
+  apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl;
+  sendNotification = title: message: ''
+    # Send notification through Apprise
+    ${pkgs.apprise}/bin/apprise -t "${title}" -b "${message}" "${apprise-url}" || true
+  '';
+
+  # Get borg stats in a readable format
+  extractBorgStats = logFile: ''
+    # Extract and format relevant stats
+    echo -e "\nBackup Stats:" > ${logFile}
+    grep -A15 "Archive name:" ${logFile} >> ${logFile}.stats || true
+    STATS=$(cat ${logFile}.stats || echo "No stats available")
+  '';
+
 in
 {
-  # Make sure borg is installed
+  # Make sure borg and apprise are installed
-  environment.systemPackages = [ pkgs.borgbackup ];
+  environment.systemPackages = with pkgs; [
+    borgbackup
+    apprise
+  ];
 
   # Docker Storage Backup Service
   systemd.services.backup-docker-storage = {
@@ -39,29 +57,62 @@ in
     path = with pkgs; [
       borgbackup
       coreutils
+      apprise
+      gnugrep
     ];
 
     script = ''
       ${borgCommonSettings}
 
+      # Set up log file and temporary stats file
+      LOG_FILE="/tmp/borg-docker-backup-$(date +%Y%m%d-%H%M%S).log"
+
+      # Send start notification
+      ${sendNotification "🚀 Docker Storage Backup Started" "Starting Docker Storage backup on $(hostname) at $(date)"}
+
       # Initialize repository if needed
       ${initRepo dockerStorageRepo}
 
-      # Create backup
+      # Create backup and capture output/stats
+      echo "Starting backup at $(date)" > $LOG_FILE
+      ARCHIVE_NAME="docker-$(date +%Y-%m-%d_%H%M%S)"
+
+      # Run backup command and capture exit status
       ${pkgs.borgbackup}/bin/borg create \
         --stats \
         --compression zstd,15 \
         --exclude '*.tmp' \
         --exclude '*/tmp/*' \
-        ${dockerStorageRepo}::docker-{now:%Y-%m-%d_%H%M%S} \
+        ${dockerStorageRepo}::$ARCHIVE_NAME \
-        /mnt/drive1/DockerStorage
+        /mnt/drive1/DockerStorage >> $LOG_FILE 2>&1
+
+      BACKUP_STATUS=$?
+
+      # Extract stats from log file
+      ${extractBorgStats "$LOG_FILE"}
 
       # Prune old backups
+      echo -e "\nPruning old backups..." >> $LOG_FILE
       ${pkgs.borgbackup}/bin/borg prune \
         --keep-daily 7 \
         --keep-weekly 4 \
         --keep-monthly 3 \
-        ${dockerStorageRepo}
+        ${dockerStorageRepo} >> $LOG_FILE 2>&1
+
+      PRUNE_STATUS=$?
+
+      # Send completion notification
+      if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then
+        ${sendNotification "✅ Docker Storage Backup Complete" "Docker Storage backup completed successfully on $(hostname) at $(date)\n\n$STATS"}
+      else
+        ${sendNotification "❌ Docker Storage Backup Failed" "Docker Storage backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nSee $LOG_FILE for details"}
+      fi
+
+      # Clean up temp stats file
+      rm -f $LOG_FILE.stats
+
+      # Return exit status from backup operation
+      exit $BACKUP_STATUS
     '';
 
     serviceConfig = {
@@ -92,29 +143,62 @@ in
     path = with pkgs; [
       borgbackup
       coreutils
+      apprise
+      gnugrep
     ];
 
     script = ''
       ${borgCommonSettings}
 
+      # Set up log file and temporary stats file
+      LOG_FILE="/tmp/borg-forgejo-backup-$(date +%Y%m%d-%H%M%S).log"
+
+      # Send start notification
+      ${sendNotification "🚀 Forgejo Backup Started" "Starting Forgejo backup on $(hostname) at $(date)"}
+
       # Initialize repository if needed
       ${initRepo forgejoRepo}
 
-      # Create backup
+      # Create backup and capture output/stats
+      echo "Starting backup at $(date)" > $LOG_FILE
+      ARCHIVE_NAME="forgejo-$(date +%Y-%m-%d_%H%M%S)"
+
+      # Run backup command and capture exit status
       ${pkgs.borgbackup}/bin/borg create \
         --stats \
         --compression zstd,15 \
         --exclude '*.tmp' \
         --exclude '*/tmp/*' \
-        ${forgejoRepo}::forgejo-{now:%Y-%m-%d_%H%M%S} \
+        ${forgejoRepo}::$ARCHIVE_NAME \
-        /pool/forgejo
+        /pool/forgejo >> $LOG_FILE 2>&1
+
+      BACKUP_STATUS=$?
+
+      # Extract stats from log file
+      ${extractBorgStats "$LOG_FILE"}
 
       # Prune old backups
+      echo -e "\nPruning old backups..." >> $LOG_FILE
       ${pkgs.borgbackup}/bin/borg prune \
         --keep-daily 14 \
         --keep-weekly 4 \
         --keep-monthly 3 \
-        ${forgejoRepo}
+        ${forgejoRepo} >> $LOG_FILE 2>&1
+
+      PRUNE_STATUS=$?
+
+      # Send completion notification
+      if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then
+        ${sendNotification "✅ Forgejo Backup Complete" "Forgejo backup completed successfully on $(hostname) at $(date)\n\n$STATS"}
+      else
+        ${sendNotification "❌ Forgejo Backup Failed" "Forgejo backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nSee $LOG_FILE for details"}
+      fi
+
+      # Clean up temp stats file
+      rm -f $LOG_FILE.stats
+
+      # Return exit status from backup operation
+      exit $BACKUP_STATUS
     '';
 
     serviceConfig = {

hosts/nixos/cloud/config/snapraid.nix

@@ -6,7 +6,7 @@
 }:
 let
 
-  apprise-url = config.secretsSpec.api.apprise-url;
+  apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl;
 
   snapraid-aio = inputs.snapraid-aio.nixosModules.default;
   snapraid-aio-config = pkgs.writeTextFile {
@@ -96,6 +96,18 @@ let
       exclude *.unrecoverable
       exclude /tmp/
       exclude /lost+found/
+      exclude /var/tmp/
+      exclude /var/cache/
+      exclude /var/log/
+      exclude .trash/
+      exclude .Trash-1000/
+      exclude .Trash/
+      # These dirs change data all the time
+      # so I back them up in borg repos that are not excluded 
+      exclude /mnt/drive1/DockerStorage/
+      exclude /mnt/drive1/data/forgejo
+      exclude /mnt/drive2/data/forgejo
+      exclude /mnt/drive3/data/forgejo
     '';
   };
 in

modules/common/secret-spec.nix

@@ -89,6 +89,45 @@ in
               description = "SSH public keys for the user";
               default = [ ];
             };
+            smtp = lib.mkOption {
+              type = lib.types.submodule {
+                options = {
+                  host = lib.mkOption {
+                    type = lib.types.str;
+                    description = "SMTP server hostname";
+                  };
+                  user = lib.mkOption {
+                    type = lib.types.str;
+                    description = "SMTP username for authentication";
+                  };
+                  password = lib.mkOption {
+                    type = lib.types.str;
+                    description = "SMTP password for authentication";
+                  };
+                  port = lib.mkOption {
+                    type = lib.types.port;
+                    description = "SMTP server port";
+                    default = 587;
+                  };
+                  from = lib.mkOption {
+                    type = lib.types.str;
+                    description = "Email address to send from";
+                  };
+                  notifyUrl = lib.mkOption {
+                    type = lib.types.str;
+                    description = "Apprise URL for sending notifications via this SMTP account";
+                    default =
+                      config:
+                      let
+                        smtp = config; # parent smtp config
+                      in
+                      "mailtos://_?user=${smtp.user}&pass=${smtp.password}&smtp=${smtp.host}&from=${smtp.from}&to=${smtp.user}";
+                  };
+                };
+              };
+              description = "SMTP configuration for the user";
+              default = null;
+            };
           };
         }
       );