diff --git a/flake.lock b/flake.lock index 67b0792..8e83f64 100644 --- a/flake.lock +++ b/flake.lock @@ -106,7 +106,7 @@ "crane": { "inputs": { "flake-compat": "flake-compat_2", - "flake-utils": "flake-utils_5", + "flake-utils": "flake-utils_6", "nixpkgs": [ "watershot", "std", @@ -308,6 +308,24 @@ } }, "flake-utils_2": { + "inputs": { + "systems": "systems_4" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_3": { "inputs": { "systems": [ "stylix", @@ -328,9 +346,9 @@ "type": "github" } }, - "flake-utils_3": { + "flake-utils_4": { "inputs": { - "systems": "systems_5" + "systems": "systems_6" }, "locked": { "lastModified": 1681202837, @@ -346,7 +364,7 @@ "type": "github" } }, - "flake-utils_4": { + "flake-utils_5": { "locked": { "lastModified": 1659877975, "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", @@ -361,7 +379,7 @@ "type": "github" } }, - "flake-utils_5": { + "flake-utils_6": { "locked": { "lastModified": 1667395993, "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", @@ -376,6 +394,24 @@ "type": "github" } }, + "flake-utils_7": { + "inputs": { + "systems": "systems_7" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "fromYaml": { "flake": false, "locked": { @@ -940,9 +976,11 @@ "nixpkgs-unstable": "nixpkgs-unstable", "nixvirt": "nixvirt", "rose-pine-hyprcursor": "rose-pine-hyprcursor", + "snapraid-aio": "snapraid-aio", "stylix": "stylix", "vscode-server": "vscode-server", "watershot": "watershot", + "yay": "yay", "zen-browser": "zen-browser" } }, @@ -1014,6 +1052,45 @@ "type": "github" } }, + "snapraid-aio": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": [ + "nixpkgs" + ], + "snapraid-aio-src": "snapraid-aio-src" + }, + "locked": { + "lastModified": 1745877167, + "narHash": "sha256-I1LF6QlQnQmpsom676VNWzbA5xY1ksgwMyPh1b5JoG0=", + "ref": "refs/heads/main", + "rev": "fb1b3606270ce8ceaea8534a046eb1dda08c54dc", + "revCount": 1, + "type": "git", + "url": "https://git.ryot.foo/toph/snapraid-aio.nix.git" + }, + "original": { + "type": "git", + "url": "https://git.ryot.foo/toph/snapraid-aio.nix.git" + } + }, + "snapraid-aio-src": { + "flake": false, + "locked": { + "lastModified": 1744884143, + "narHash": "sha256-GNXn/V4HoFnQtyq7l+V+aXHArObr3zQd4vCgPEqPeRk=", + "owner": "auanasgheps", + "repo": "snapraid-aio-script", + "rev": "a46c7362af385eac945e86a2a0f6097dbe7ca3fb", + "type": "github" + }, + "original": { + "owner": "auanasgheps", + "repo": "snapraid-aio-script", + "rev": "a46c7362af385eac945e86a2a0f6097dbe7ca3fb", + "type": "github" + } + }, "std": { "inputs": { "arion": [ @@ -1024,7 +1101,7 @@ "blank": "blank", "devshell": "devshell", "dmerge": "dmerge", - "flake-utils": "flake-utils_4", + "flake-utils": "flake-utils_5", "incl": "incl", "makes": [ "watershot", @@ -1069,13 +1146,13 @@ "base16-vim": "base16-vim", "firefox-gnome-theme": "firefox-gnome-theme", "flake-compat": "flake-compat", - "flake-utils": "flake-utils_2", + "flake-utils": "flake-utils_3", "git-hooks": "git-hooks", "gnome-shell": "gnome-shell", "home-manager": "home-manager_2", "nixpkgs": "nixpkgs_3", "nur": "nur", - "systems": "systems_4", + "systems": "systems_5", "tinted-foot": "tinted-foot", "tinted-kitty": "tinted-kitty", "tinted-schemes": "tinted-schemes", @@ -1171,6 +1248,36 @@ "type": "github" } }, + "systems_6": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_7": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, "tinted-foot": { "flake": false, "locked": { @@ -1295,7 +1402,7 @@ }, "vscode-server": { "inputs": { - "flake-utils": "flake-utils_3", + "flake-utils": "flake-utils_4", "nixpkgs": [ "nixpkgs-unstable" ] @@ -1358,6 +1465,27 @@ "type": "github" } }, + "yay": { + "inputs": { + "flake-utils": "flake-utils_7", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1745989032, + "narHash": "sha256-qKy5YVu8vhA60VxWpLiLV9QpN8LofL9qFCEAACrCxBw=", + "ref": "refs/heads/main", + "rev": "92d557d0d0393713cb57a970e880efafe6cc2b41", + "revCount": 9, + "type": "git", + "url": "https://git.ryot.foo/toph/yay.nix.git" + }, + "original": { + "type": "git", + "url": "https://git.ryot.foo/toph/yay.nix.git" + } + }, "zen-browser": { "inputs": { "nixpkgs": [ diff --git a/hosts/nixos/cloud/config/borg.nix b/hosts/nixos/cloud/config/borg.nix index 2e12aae..07bd3ea 100644 --- a/hosts/nixos/cloud/config/borg.nix +++ b/hosts/nixos/cloud/config/borg.nix @@ -6,20 +6,29 @@ }: let - # Borg backup destinations + # Common repositories dockerStorageRepo = "/pool/Backups/DockerStorage"; forgejoRepo = "/pool/Backups/forgejo"; - # Common borg backup settings + # Shared environment setup borgCommonSettings = '' # Don't use cache to avoid issues with concurrent backups export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes - - # Set this for non-interactive use export BORG_NON_INTERACTIVE=yes ''; - # Initialize a repo if it doesn't exist + # Common packages needed for backups + commonBorgPath = with pkgs; [ + borgbackup + coreutils + apprise + gnugrep + hostname + util-linux + gawk + ]; + + # Repository initialization initRepo = repo: '' if [ ! -d "${repo}" ]; then mkdir -p "${repo}" @@ -27,114 +36,145 @@ let fi ''; + # Notification system + apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl; + sendNotification = title: message: '' + ${pkgs.apprise}/bin/apprise -t "${title}" -b "${message}" "${apprise-url}" || true + ''; + + # Statistics generation + extractBorgStats = logFile: repoPath: '' + { + echo -e "\n==== BACKUP SUMMARY ====\n" + grep -A10 "Archive name:" ${logFile} || echo "No archive stats found" + echo -e "\n=== Compression ===\n" + grep "Compressed size:" ${logFile} || echo "No compression stats found" + echo -e "\n=== Duration ===\n" + grep "Duration:" ${logFile} || echo "No duration stats found" + grep "Throughput:" ${logFile} || echo "No throughput stats found" + echo -e "\n=== Repository ===\n" + ${pkgs.borgbackup}/bin/borg info ${repoPath} --last 1 2>/dev/null || echo "Could not get repository info" + echo -e "\n=== Storage Space ===\n" + df -h ${repoPath} | grep -v "Filesystem" || echo "Could not get storage info" + } > ${logFile}.stats + STATS=$(cat ${logFile}.stats || echo "No stats available") + ''; + + # Unified backup service generator + mkBorgBackupService = + { + name, + title, + repo, + sourcePath, + keepDaily, + keepWeekly, + keepMonthly, + schedule, + }: + { + services."backup-${name}" = { + description = "Backup ${title} with Borg"; + inherit (commonServiceConfig) path serviceConfig; + + script = '' + ${borgCommonSettings} + + LOG_FILE="/tmp/borg-${name}-backup-$(date +%Y%m%d-%H%M%S).log" + ${initRepo repo} + + echo "Starting ${title} backup at $(date)" > $LOG_FILE + ARCHIVE_NAME="${name}-$(date +%Y-%m-%d_%H%M%S)" + START_TIME=$(date +%s) + + ${pkgs.borgbackup}/bin/borg create \ + --stats \ + --compression zstd,15 \ + --exclude '*.tmp' \ + --exclude '*/tmp/*' \ + ${repo}::$ARCHIVE_NAME \ + ${sourcePath} >> $LOG_FILE 2>&1 + + BACKUP_STATUS=$? + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + echo "Total time: $DURATION seconds ($(date -d@$DURATION -u +%H:%M:%S))" >> $LOG_FILE + + ${extractBorgStats "$LOG_FILE" "${repo}"} + + echo -e "\nPruning old backups..." >> $LOG_FILE + ${pkgs.borgbackup}/bin/borg prune \ + --keep-daily ${toString keepDaily} \ + --keep-weekly ${toString keepWeekly} \ + --keep-monthly ${toString keepMonthly} \ + ${repo} >> $LOG_FILE 2>&1 + + PRUNE_STATUS=$? + + echo -e "\nRemaining archives after pruning:" >> $LOG_FILE + ${pkgs.borgbackup}/bin/borg list ${repo} >> $LOG_FILE 2>&1 || true + + if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then + ${sendNotification "✅ ${title} Backup Complete" "${title} backup completed successfully on $(hostname) at $(date)\nDuration: $(date -d@$DURATION -u +%H:%M:%S)\n\n$STATS"} + else + ${sendNotification "❌ ${title} Backup Failed" "${title} backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nPartial Stats:\n$STATS\n\nSee $LOG_FILE for details"} + fi + + rm -f $LOG_FILE.stats + exit $BACKUP_STATUS + ''; + }; + + timers."backup-${name}" = { + description = "Timer for ${title} Backup"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = schedule; + Persistent = true; + RandomizedDelaySec = "5min"; + }; + }; + }; + + # Common service configuration + commonServiceConfig = { + path = commonBorgPath; + serviceConfig = { + Type = "oneshot"; + IOSchedulingClass = "idle"; + CPUSchedulingPolicy = "idle"; + Nice = 19; + }; + }; + in { - # Make sure borg is installed - environment.systemPackages = [ pkgs.borgbackup ]; + environment.systemPackages = with pkgs; [ + borgbackup + apprise + ]; - # Docker Storage Backup Service - systemd.services.backup-docker-storage = { - description = "Backup Docker storage directory with Borg"; + systemd = lib.mkMerge [ + (mkBorgBackupService { + name = "docker-storage"; + title = "Docker Storage"; + repo = dockerStorageRepo; + sourcePath = "/mnt/drive1/DockerStorage"; + keepDaily = 7; + keepWeekly = 4; + keepMonthly = 3; + schedule = "Mon *-*-* 04:00:00"; + }) - path = with pkgs; [ - borgbackup - coreutils - ]; - - script = '' - ${borgCommonSettings} - - # Initialize repository if needed - ${initRepo dockerStorageRepo} - - # Create backup - ${pkgs.borgbackup}/bin/borg create \ - --stats \ - --compression zstd,15 \ - --exclude '*.tmp' \ - --exclude '*/tmp/*' \ - ${dockerStorageRepo}::docker-{now:%Y-%m-%d_%H%M%S} \ - /mnt/drive1/DockerStorage - - # Prune old backups - ${pkgs.borgbackup}/bin/borg prune \ - --keep-daily 7 \ - --keep-weekly 4 \ - --keep-monthly 3 \ - ${dockerStorageRepo} - ''; - - serviceConfig = { - Type = "oneshot"; - IOSchedulingClass = "idle"; - CPUSchedulingPolicy = "idle"; - Nice = 19; - }; - }; - - # Docker Storage Backup Timer (Weekly on Monday at 4am) - systemd.timers.backup-docker-storage = { - description = "Timer for Docker Storage Backup"; - - wantedBy = [ "timers.target" ]; - - timerConfig = { - OnCalendar = "Mon *-*-* 04:00:00"; - Persistent = true; # Run backup if system was off during scheduled time - RandomizedDelaySec = "5min"; # Add randomized delay - }; - }; - - # Forgejo Backup Service - systemd.services.backup-forgejo = { - description = "Backup Forgejo directory with Borg"; - - path = with pkgs; [ - borgbackup - coreutils - ]; - - script = '' - ${borgCommonSettings} - - # Initialize repository if needed - ${initRepo forgejoRepo} - - # Create backup - ${pkgs.borgbackup}/bin/borg create \ - --stats \ - --compression zstd,15 \ - --exclude '*.tmp' \ - --exclude '*/tmp/*' \ - ${forgejoRepo}::forgejo-{now:%Y-%m-%d_%H%M%S} \ - /pool/forgejo - - # Prune old backups - ${pkgs.borgbackup}/bin/borg prune \ - --keep-daily 14 \ - --keep-weekly 4 \ - --keep-monthly 3 \ - ${forgejoRepo} - ''; - - serviceConfig = { - Type = "oneshot"; - IOSchedulingClass = "idle"; - CPUSchedulingPolicy = "idle"; - Nice = 19; - }; - }; - - # Forgejo Backup Timer (Every 2 days at 4am) - systemd.timers.backup-forgejo = { - description = "Timer for Forgejo Backup"; - - wantedBy = [ "timers.target" ]; - - timerConfig = { - OnCalendar = "*-*-1/2 04:00:00"; # Every 2 days at 4am - Persistent = true; - RandomizedDelaySec = "5min"; - }; - }; + (mkBorgBackupService { + name = "forgejo"; + title = "Forgejo"; + repo = forgejoRepo; + sourcePath = "/pool/forgejo"; + keepDaily = 14; + keepWeekly = 4; + keepMonthly = 3; + schedule = "*-*-1/2 04:00:00"; + }) + ]; } diff --git a/hosts/nixos/cloud/config/snapraid.nix b/hosts/nixos/cloud/config/snapraid.nix index fa2b31e..a62cfcb 100644 --- a/hosts/nixos/cloud/config/snapraid.nix +++ b/hosts/nixos/cloud/config/snapraid.nix @@ -6,7 +6,7 @@ }: let - apprise-url = config.secretsSpec.api.apprise-url; + apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl; snapraid-aio = inputs.snapraid-aio.nixosModules.default; snapraid-aio-config = pkgs.writeTextFile { @@ -96,6 +96,18 @@ let exclude *.unrecoverable exclude /tmp/ exclude /lost+found/ + exclude /var/tmp/ + exclude /var/cache/ + exclude /var/log/ + exclude .trash/ + exclude .Trash-1000/ + exclude .Trash/ + # These dirs change data all the time + # so I back them up in borg repos that are not excluded + exclude /mnt/drive1/DockerStorage/ + exclude /mnt/drive1/data/forgejo + exclude /mnt/drive2/data/forgejo + exclude /mnt/drive3/data/forgejo ''; }; in diff --git a/modules/common/secret-spec.nix b/modules/common/secret-spec.nix index 02dca79..000f04c 100644 --- a/modules/common/secret-spec.nix +++ b/modules/common/secret-spec.nix @@ -18,6 +18,17 @@ let grep -q "BEGIN OPENSSH PRIVATE KEY" "$out" || (echo "Invalid SSH key format"; exit 1) ''; }; + + # Function to build an Apprise URL from SMTP settings + buildAppriseUrl = + { + host, + user, + password, + from, + ... + }: + "mailtos://_?user=${user}&pass=${password}&smtp=${host}&from=${from}&to=${user}"; in { options.secretsSpec = { @@ -89,6 +100,45 @@ in description = "SSH public keys for the user"; default = [ ]; }; + smtp = lib.mkOption { + type = lib.types.submodule ( + { config, ... }: + { + options = { + host = lib.mkOption { + type = lib.types.str; + description = "SMTP server hostname"; + }; + user = lib.mkOption { + type = lib.types.str; + description = "SMTP username for authentication"; + }; + password = lib.mkOption { + type = lib.types.str; + description = "SMTP password for authentication"; + }; + port = lib.mkOption { + type = lib.types.port; + description = "SMTP server port"; + default = 587; + }; + from = lib.mkOption { + type = lib.types.str; + description = "Email address to send from"; + }; + notifyUrl = lib.mkOption { + type = lib.types.str; + description = "Apprise URL for sending notifications via this SMTP account"; + }; + }; + config = { + notifyUrl = "mailtos://_?user=${config.user}&pass=${config.password}&smtp=${config.host}&from=${config.from}&to=${config.user}"; + }; + } + ); + description = "SMTP configuration for the user"; + default = null; + }; }; } ); diff --git a/secrets.nix b/secrets.nix index 7d90baa..1ea522d 100644 Binary files a/secrets.nix and b/secrets.nix differ