dot.nix/hosts/nixos/cloud/config/borg.nix

{
  config,
  lib,
  pkgs,
  ...
}:

let
  # Borg backup destinations
  dockerStorageRepo = "/pool/Backups/DockerStorage";
  forgejoRepo = "/pool/Backups/forgejo";

  # Common borg backup settings
  borgCommonSettings = ''
    # Don't use cache to avoid issues with concurrent backups
    export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes

    # Set this for non-interactive use
    export BORG_NON_INTERACTIVE=yes
  '';

  # Initialize a repo if it doesn't exist
  initRepo = repo: ''
    if [ ! -d "${repo}" ]; then
      mkdir -p "${repo}"
      ${pkgs.borgbackup}/bin/borg init --encryption=none "${repo}"
    fi
  '';

  # Function to send notifications
  apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl;
  sendNotification = title: message: ''
    # Send notification through Apprise
    ${pkgs.apprise}/bin/apprise -t "${title}" -b "${message}" "${apprise-url}" || true
  '';

  # Get borg stats in a readable format
  extractBorgStats = logFile: ''
    # Extract and format relevant stats
    echo -e "\nBackup Stats:" > ${logFile}
    grep -A15 "Archive name:" ${logFile} >> ${logFile}.stats || true
    STATS=$(cat ${logFile}.stats || echo "No stats available")
  '';

in
{
  # Make sure borg and apprise are installed
  environment.systemPackages = with pkgs; [
    borgbackup
    apprise
  ];

  # Docker Storage Backup Service
  systemd.services.backup-docker-storage = {
    description = "Backup Docker storage directory with Borg";

    path = with pkgs; [
      borgbackup
      coreutils
      apprise
      gnugrep
    ];

    script = ''
      ${borgCommonSettings}

      # Set up log file and temporary stats file
      LOG_FILE="/tmp/borg-docker-backup-$(date +%Y%m%d-%H%M%S).log"

      # Send start notification
      ${sendNotification "🚀 Docker Storage Backup Started" "Starting Docker Storage backup on $(hostname) at $(date)"}

      # Initialize repository if needed
      ${initRepo dockerStorageRepo}

      # Create backup and capture output/stats
      echo "Starting backup at $(date)" > $LOG_FILE
      ARCHIVE_NAME="docker-$(date +%Y-%m-%d_%H%M%S)"

      # Run backup command and capture exit status
      ${pkgs.borgbackup}/bin/borg create \
        --stats \
        --compression zstd,15 \
        --exclude '*.tmp' \
        --exclude '*/tmp/*' \
        ${dockerStorageRepo}::$ARCHIVE_NAME \
        /mnt/drive1/DockerStorage >> $LOG_FILE 2>&1

      BACKUP_STATUS=$?

      # Extract stats from log file
      ${extractBorgStats "$LOG_FILE"}

      # Prune old backups
      echo -e "\nPruning old backups..." >> $LOG_FILE
      ${pkgs.borgbackup}/bin/borg prune \
        --keep-daily 7 \
        --keep-weekly 4 \
        --keep-monthly 3 \
        ${dockerStorageRepo} >> $LOG_FILE 2>&1

      PRUNE_STATUS=$?

      # Send completion notification
      if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then
        ${sendNotification "✅ Docker Storage Backup Complete" "Docker Storage backup completed successfully on $(hostname) at $(date)\n\n$STATS"}
      else
        ${sendNotification "❌ Docker Storage Backup Failed" "Docker Storage backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nSee $LOG_FILE for details"}
      fi

      # Clean up temp stats file
      rm -f $LOG_FILE.stats

      # Return exit status from backup operation
      exit $BACKUP_STATUS
    '';

    serviceConfig = {
      Type = "oneshot";
      IOSchedulingClass = "idle";
      CPUSchedulingPolicy = "idle";
      Nice = 19;
    };
  };

  # Docker Storage Backup Timer (Weekly on Monday at 4am)
  systemd.timers.backup-docker-storage = {
    description = "Timer for Docker Storage Backup";

    wantedBy = [ "timers.target" ];

    timerConfig = {
      OnCalendar = "Mon *-*-* 04:00:00";
      Persistent = true; # Run backup if system was off during scheduled time
      RandomizedDelaySec = "5min"; # Add randomized delay
    };
  };

  # Forgejo Backup Service
  systemd.services.backup-forgejo = {
    description = "Backup Forgejo directory with Borg";

    path = with pkgs; [
      borgbackup
      coreutils
      apprise
      gnugrep
    ];

    script = ''
      ${borgCommonSettings}

      # Set up log file and temporary stats file
      LOG_FILE="/tmp/borg-forgejo-backup-$(date +%Y%m%d-%H%M%S).log"

      # Send start notification
      ${sendNotification "🚀 Forgejo Backup Started" "Starting Forgejo backup on $(hostname) at $(date)"}

      # Initialize repository if needed
      ${initRepo forgejoRepo}

      # Create backup and capture output/stats
      echo "Starting backup at $(date)" > $LOG_FILE
      ARCHIVE_NAME="forgejo-$(date +%Y-%m-%d_%H%M%S)"

      # Run backup command and capture exit status
      ${pkgs.borgbackup}/bin/borg create \
        --stats \
        --compression zstd,15 \
        --exclude '*.tmp' \
        --exclude '*/tmp/*' \
        ${forgejoRepo}::$ARCHIVE_NAME \
        /pool/forgejo >> $LOG_FILE 2>&1

      BACKUP_STATUS=$?

      # Extract stats from log file
      ${extractBorgStats "$LOG_FILE"}

      # Prune old backups
      echo -e "\nPruning old backups..." >> $LOG_FILE
      ${pkgs.borgbackup}/bin/borg prune \
        --keep-daily 14 \
        --keep-weekly 4 \
        --keep-monthly 3 \
        ${forgejoRepo} >> $LOG_FILE 2>&1

      PRUNE_STATUS=$?

      # Send completion notification
      if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then
        ${sendNotification "✅ Forgejo Backup Complete" "Forgejo backup completed successfully on $(hostname) at $(date)\n\n$STATS"}
      else
        ${sendNotification "❌ Forgejo Backup Failed" "Forgejo backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nSee $LOG_FILE for details"}
      fi

      # Clean up temp stats file
      rm -f $LOG_FILE.stats

      # Return exit status from backup operation
      exit $BACKUP_STATUS
    '';

    serviceConfig = {
      Type = "oneshot";
      IOSchedulingClass = "idle";
      CPUSchedulingPolicy = "idle";
      Nice = 19;
    };
  };

  # Forgejo Backup Timer (Every 2 days at 4am)
  systemd.timers.backup-forgejo = {
    description = "Timer for Forgejo Backup";

    wantedBy = [ "timers.target" ];

    timerConfig = {
      OnCalendar = "*-*-1/2 04:00:00"; # Every 2 days at 4am
      Persistent = true;
      RandomizedDelaySec = "5min";
    };
  };
}