hosts/common/optional/docker.nix

@@ -7,8 +7,4 @@
     };
     oci-containers.backend = "docker";
   };
-
-  environment.systemPackages = with pkgs; [
-    lazydocker
-  ];
 }

hosts/nixos/cloud/config/backups/default.nix

@@ -1,197 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-
-let
-  # Shared configuration
-  logDir = "/var/log/backups";
-  backupServices = [
-    {
-      name = "forgejo";
-      title = "Forgejo";
-      service = "backup-forgejo.service";
-      logPattern = "borg-forgejo-backup-*.log";
-    }
-    {
-      name = "docker_storage";
-      title = "Docker Storage";
-      service = "backup-docker-storage.service";
-      logPattern = "borg-docker-storage-backup-*.log";
-    }
-    {
-      name = "snapraid";
-      title = "SnapRAID";
-      service = "snapraid-aio.service";
-      logPattern = "SnapRAID-*.out";
-      logPath = "/var/log/snapraid";
-    }
-  ];
-
-  # Helper functions
-  users = config.secretsSpec.users;
-  notify =
-    title: message: logFile:
-    let
-      attachArg = if logFile == "" then "" else "--attach \"file://${logFile}\"";
-      appriseUrl = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo";
-    in
-    ''
-      ${pkgs.apprise}/bin/apprise  -vv -i "markdown" -t "${title}" \
-        -b "${message}" \
-        ${attachArg} \
-        "${appriseUrl}" || true
-    '';
-
-  findLatestLog = pattern: path: ''
-    find "${path}" -name "${pattern}" -type f -printf "%T@ %p\\n" 2>/dev/null \
-      | sort -nr | head -1 | cut -d' ' -f2
-  '';
-
-  # Generate safe variable name (replace hyphens with underscores)
-  safeName = name: lib.replaceStrings [ "-" ] [ "_" ] name;
-
-  # Generate status variable references
-  statusVarName = name: "STATUS_${safeName name}";
-
-  # Common script utilities
-  scriptPrelude = ''
-    set -uo pipefail
-    LOG_FILE="${logDir}/backup-chain-$(date +%Y%m%d-%H%M%S).log"
-    mkdir -p "${logDir}"
-    exec > >(tee -a "$LOG_FILE") 2>&1
-
-    log() {
-      echo "[$(date "+%Y-%m-%d %H:%M:%S")] $1"
-    }
-
-    # Initialize all status variables
-    ${lib.concatMapStringsSep "\n" (s: "${statusVarName s.name}=1") backupServices}
-  '';
-
-  # Service runner template
-  runService =
-    {
-      name,
-      title,
-      service,
-      logPattern,
-      logPath ? "/tmp",
-    }:
-    ''
-      log "Starting ${title} maintenance..."
-      systemctl start ${service} || true
-      ${statusVarName name}=$?
-      log "${title} completed with status $${statusVarName name}"
-
-      SERVICE_LOG=$(${findLatestLog logPattern logPath})
-      if [ -n "$SERVICE_LOG" ]; then
-        log "Appending ${title} log: $SERVICE_LOG"
-        echo -e "\n\n===== ${title} LOG ($(basename "$SERVICE_LOG")) =====\n" >> "$LOG_FILE"
-        cat "$SERVICE_LOG" >> "$LOG_FILE"
-        
-        # Add SnapRAID-specific summary
-        if [ "${name}" = "snapraid" ]; then
-          echo -e "\n=== SnapRAID Summary ===" >> "$LOG_FILE"
-          grep -E '(Scrub|Sync|Diff|smart)' "$SERVICE_LOG" | tail -n 10 >> "$LOG_FILE"
-        fi
-      fi
-    '';
-
-  # Build the service execution script
-  serviceExecution = lib.concatMapStrings runService backupServices;
-
-  # Generate status summary lines
-  statusSummaryLines = lib.concatMapStringsSep "\n" (
-    s:
-    let
-      varName = statusVarName s.name;
-    in
-    "- **${s.title}:** \$([ \$${varName} -eq 0 ] && echo '✅ Success' || echo '❌ Failed') (Exit: \$${varName})"
-  ) backupServices;
-
-  # Notification logic with cleaner formatting
-  notificationLogic =
-    let
-      statusVars = map (s: statusVarName s.name) backupServices;
-      statusChecks = lib.concatMapStringsSep "\n" (var: "[ \$${var} -eq 0 ] && ") statusVars;
-    in
-    ''
-      # Calculate overall status
-      OVERALL_STATUS=0
-      ${lib.concatMapStringsSep "\n" (var: "if [ \$${var} -ne 0 ]; then OVERALL_STATUS=1; fi") statusVars}
-
-      TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
-      HOSTNAME=$(hostname)
-
-      SUMMARY=$(cat << EOF
-      # Backup Chain Complete
-
-      **Host:** $HOSTNAME  
-      **Timestamp:** $TIMESTAMP  
-      **Overall Status:** $([ $OVERALL_STATUS -eq 0 ] && echo '✅ Success' || echo '⚠️ Failure')
-
-      ## Service Status:
-      ${statusSummaryLines}
-
-      **Log Path:** $LOG_FILE
-      EOF)
-
-      if [ $OVERALL_STATUS -eq 0 ]; then
-        ${notify "✅ Backup Success" "$SUMMARY" "$LOG_FILE"}
-      else
-        ${notify "⚠️ Backup Issues" "$SUMMARY" "$LOG_FILE"}
-      fi
-
-      exit $OVERALL_STATUS
-    '';
-
-in
-{
-  imports = lib.custom.scanPaths ./.;
-
-  systemd.services.backup-chain = {
-    description = "Orchestrated Backup Chain";
-    path = with pkgs; [
-      apprise
-      coreutils
-      findutils
-      gawk
-      gnugrep
-      hostname
-      systemd
-      util-linux
-    ];
-
-    serviceConfig = {
-      Type = "oneshot";
-      Nice = 19;
-      IOSchedulingClass = "idle";
-      CPUSchedulingPolicy = "idle";
-    };
-
-    script = ''
-      ${scriptPrelude}
-      log "Initializing backup chain on $(hostname)"
-
-      ${serviceExecution}
-
-      log "Finalizing backup chain"
-      ${notificationLogic}
-    '';
-  };
-
-  systemd.timers.backup-chain = {
-    wantedBy = [ "timers.target" ];
-    timerConfig = {
-      OnCalendar = "*-*-* 03:00:00";
-      Persistent = true;
-      RandomizedDelaySec = "5min";
-    };
-  };
-
-  environment.systemPackages = [ pkgs.apprise ];
-  systemd.tmpfiles.rules = [ "d ${logDir} 0755 root root -" ];
-}

hosts/nixos/cloud/config/borg.nix

@@ -60,7 +60,7 @@ let
     STATS=$(cat ${logFile}.stats || echo "No stats available")
   '';
 
-  # Unified backup service generator with optional features
+  # Unified backup service generator
   mkBorgBackupService =
     {
       name,
@@ -70,25 +70,8 @@ let
       keepDaily,
       keepWeekly,
       keepMonthly,
-      schedule ? null,
+      schedule,
-      enableNotifications ? true,
-      verbose ? false,
     }:
-    let
-      maybeCreateTimer = lib.optionalAttrs (schedule != null) {
-        timers."backup-${name}" = {
-          description = "Timer for ${title} Backup";
-          wantedBy = [ "timers.target" ];
-          timerConfig = {
-            OnCalendar = schedule;
-            Persistent = true;
-            RandomizedDelaySec = "5min";
-          };
-        };
-      };
-
-      logPrefix = if verbose then "set -x;" else "";
-    in
     {
       services."backup-${name}" = {
         description = "Backup ${title} with Borg";
@@ -96,7 +79,6 @@ let
 
         script = ''
           ${borgCommonSettings}
-          ${logPrefix}  # Add verbose logging if enabled
 
           LOG_FILE="/tmp/borg-${name}-backup-$(date +%Y%m%d-%H%M%S).log"
           ${initRepo repo}
@@ -105,15 +87,13 @@ let
           ARCHIVE_NAME="${name}-$(date +%Y-%m-%d_%H%M%S)"
           START_TIME=$(date +%s)
 
-          # Add verbose output redirection if enabled
-          ${if verbose then "exec 3>&1 4>&2" else ""}
           ${pkgs.borgbackup}/bin/borg create \
             --stats \
             --compression zstd,15 \
             --exclude '*.tmp' \
             --exclude '*/tmp/*' \
             ${repo}::$ARCHIVE_NAME \
-            ${sourcePath} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""}
+            ${sourcePath} >> $LOG_FILE 2>&1
 
           BACKUP_STATUS=$?
           END_TIME=$(date +%s)
@@ -127,33 +107,34 @@ let
             --keep-daily ${toString keepDaily} \
             --keep-weekly ${toString keepWeekly} \
             --keep-monthly ${toString keepMonthly} \
-            ${repo} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""}
+            ${repo} >> $LOG_FILE 2>&1
 
           PRUNE_STATUS=$?
 
           echo -e "\nRemaining archives after pruning:" >> $LOG_FILE
           ${pkgs.borgbackup}/bin/borg list ${repo} >> $LOG_FILE 2>&1 || true
 
-          ${
-            if enableNotifications then
-              ''
           if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then
             ${sendNotification "✅ ${title} Backup Complete" "${title} backup completed successfully on $(hostname) at $(date)\nDuration: $(date -d@$DURATION -u +%H:%M:%S)\n\n$STATS"}
           else
             ${sendNotification "❌ ${title} Backup Failed" "${title} backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nPartial Stats:\n$STATS\n\nSee $LOG_FILE for details"}
           fi
-              ''
-            else
-              "echo 'Notifications disabled' >> $LOG_FILE"
-          }
 
           rm -f $LOG_FILE.stats
           exit $BACKUP_STATUS
         '';
       };
 
-    }
+      timers."backup-${name}" = {
-    // maybeCreateTimer;
+        description = "Timer for ${title} Backup";
+        wantedBy = [ "timers.target" ];
+        timerConfig = {
+          OnCalendar = schedule;
+          Persistent = true;
+          RandomizedDelaySec = "5min";
+        };
+      };
+    };
 
   # Common service configuration
   commonServiceConfig = {
@@ -170,6 +151,7 @@ in
 {
   environment.systemPackages = with pkgs; [
     borgbackup
+    apprise
   ];
 
   systemd = lib.mkMerge [
@@ -178,17 +160,10 @@ in
       title = "Docker Storage";
       repo = dockerStorageRepo;
       sourcePath = "/mnt/drive1/DockerStorage";
-      # INFO: This shit confusing but basically
-      # keeps the last 7 days,
-      # then keeps AT LEAST ONE for last 4 weeks
-      # and finally AT LEAST ONE for the last 3 months
       keepDaily = 7;
       keepWeekly = 4;
       keepMonthly = 3;
-      # No schedule = no timer created
+      schedule = "Mon *-*-* 04:00:00";
-      # schedule = "*-*-* 03:00:00";
-      enableNotifications = false;
-      verbose = true;
     })
 
     (mkBorgBackupService {
@@ -196,12 +171,10 @@ in
       title = "Forgejo";
       repo = forgejoRepo;
       sourcePath = "/pool/forgejo";
-      keepDaily = 7;
+      keepDaily = 14;
       keepWeekly = 4;
       keepMonthly = 3;
-      # schedule = "*-*-* 03:00:00";
+      schedule = "*-*-1/2 04:00:00";
-      enableNotifications = false;
-      verbose = true;
     })
   ];
 }

hosts/nixos/cloud/config/snapraid.nix

@@ -1,13 +1,12 @@
 {
   pkgs,
   inputs,
-  lib,
   config,
   ...
 }:
 let
-  users = config.secretsSpec.users;
+
-  apprise-url = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo";
+  apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl;
 
   snapraid-aio = inputs.snapraid-aio.nixosModules.default;
   snapraid-aio-config = pkgs.writeTextFile {
@@ -21,7 +20,7 @@ let
       APPRISE_URL=""
       APPRISE_ATTACH=1
       APPRISE_BIN="${pkgs.apprise}/bin/apprise"
-      APPRISE_EMAIL=0
+      APPRISE_EMAIL=1
       APPRISE_EMAIL_URL="${apprise-url}"
       TELEGRAM=0
       DISCORD=0
@@ -105,10 +104,10 @@ let
       exclude .Trash/
       # These dirs change data all the time
       # so I back them up in borg repos that are not excluded 
-      exclude /DockerStorage/
+      exclude /mnt/drive1/DockerStorage/
-      exclude /data/forgejo/
+      exclude /mnt/drive1/data/forgejo
-      exclude /data/forgejo/
+      exclude /mnt/drive2/data/forgejo
-      exclude /data/forgejo/
+      exclude /mnt/drive3/data/forgejo
     '';
   };
 in
@@ -117,19 +116,19 @@ in
     inputs.snapraid-aio.nixosModules.default
   ];
 
+  # Make sure the SnapRAID config exists
+  environment.etc."snapraid.conf".source = snapraid-conf;
+
   # Create required directories
   systemd.tmpfiles.rules = [
     "d /var/lib/snapraid-aio 0755 root root -"
     "d /var/log/snapraid 0755 root root -"
   ];
 
-  environment.systemPackages = [ pkgs.snapraid ];
-  environment.etc."snapraid.conf".source = snapraid-conf;
-
   # Set up snapraid-aio service
   services.snapraid-aio = {
     enable = true;
     configFile = snapraid-aio-config;
-    # schedule = "*-*-* 04:00:00"; # Run daily at 3am
+    schedule = "*-*-* 03:00:00"; # Run daily at 3am
   };
 }

hosts/nixos/cloud/default.nix

@@ -60,7 +60,10 @@ in
   ## System-wide packages ##
   programs.nix-ld.enable = true;
   environment.systemPackages = with pkgs; [
+    apprise
+    lazydocker
     mergerfs
+    snapraid
   ];
 
   # https://wiki.nixos.org/wiki/FAQ/When_do_I_update_stateVersion

lib/default.nix

@@ -18,20 +18,4 @@
         ) (builtins.readDir path)
       )
     );
-
-  # Generate an Apprise URL for sending notifications
-  # Can be called with smtp config and recipient:
-  # mkAppriseUrl smtpConfig recipient
-  # Or with individual parameters:
-  # mkAppriseUrl { user = "user"; password = "pass"; host = "smtp.example.com"; from = "sender@example.com"; } "recipient@example.com"
-  mkAppriseUrl =
-    smtp: recipient:
-    let
-      smtpUser = if builtins.isAttrs smtp then smtp.user else smtp;
-      smtpPass = if builtins.isAttrs smtp then smtp.password else recipient;
-      smtpHost = if builtins.isAttrs smtp then smtp.host else "";
-      smtpFrom = if builtins.isAttrs smtp then smtp.from else "";
-      to = if builtins.isAttrs smtp then recipient else smtp.user;
-    in
-    "mailtos://_?user=${smtpUser}&pass=${smtpPass}&smtp=${smtpHost}&from=${smtpFrom}&to=${to}";
 }

modules/common/secret-spec.nix

@@ -18,6 +18,17 @@ let
         grep -q "BEGIN OPENSSH PRIVATE KEY" "$out" || (echo "Invalid SSH key format"; exit 1)
       '';
     };
+
+  # Function to build an Apprise URL from SMTP settings
+  buildAppriseUrl =
+    {
+      host,
+      user,
+      password,
+      from,
+      ...
+    }:
+    "mailtos://_?user=${user}&pass=${password}&smtp=${host}&from=${from}&to=${user}";
 in
 {
   options.secretsSpec = {
@@ -115,6 +126,13 @@ in
                       type = lib.types.str;
                       description = "Email address to send from";
                     };
+                    notifyUrl = lib.mkOption {
+                      type = lib.types.str;
+                      description = "Apprise URL for sending notifications via this SMTP account";
+                    };
+                  };
+                  config = {
+                    notifyUrl = "mailtos://_?user=${config.user}&pass=${config.password}&smtp=${config.host}&from=${config.from}&to=${config.user}";
                   };
                 }
               );