dot.nix/hosts/nixos/cloud/config/backups/default.nix

{
  config,
  lib,
  pkgs,
  ...
}:

let
  # Shared configuration
  logDir = "/var/log/backups";
  backupServices = [
    {
      name = "forgejo";
      title = "Forgejo";
      service = "backup-forgejo.service";
      logPattern = "borg-forgejo-backup-*.log";
    }
    {
      name = "docker_storage";
      title = "Docker Storage";
      service = "backup-docker-storage.service";
      logPattern = "borg-docker-storage-backup-*.log";
    }
    {
      name = "snapraid";
      title = "SnapRAID";
      service = "snapraid-aio.service";
      logPattern = "SnapRAID-*.out";
      logPath = "/var/log/snapraid";
    }
  ];

  # Helper functions
  users = config.secretsSpec.users;
  notify =
    title: message: logFile:
    let
      attachArg = if logFile == "" then "" else "--attach \"file://${logFile}\"";
      appriseUrl = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo";
    in
    ''
      ${pkgs.apprise}/bin/apprise  -vv -i "markdown" -t "${title}" \
        -b "${message}" \
        ${attachArg} \
        "${appriseUrl}" || true
    '';

  findLatestLog = pattern: path: ''
    find "${path}" -name "${pattern}" -type f -printf "%T@ %p\\n" 2>/dev/null \
      | sort -nr | head -1 | cut -d' ' -f2
  '';

  # Generate safe variable name (replace hyphens with underscores)
  safeName = name: lib.replaceStrings [ "-" ] [ "_" ] name;

  # Generate status variable references
  statusVarName = name: "STATUS_${safeName name}";

  # Common script utilities
  scriptPrelude = ''
    set -uo pipefail
    LOG_FILE="${logDir}/backup-chain-$(date +%Y%m%d-%H%M%S).log"
    mkdir -p "${logDir}"
    exec > >(tee -a "$LOG_FILE") 2>&1

    log() {
      echo "[$(date "+%Y-%m-%d %H:%M:%S")] $1"
    }

    # Initialize all status variables
    ${lib.concatMapStringsSep "\n" (s: "${statusVarName s.name}=1") backupServices}
  '';

  # Service runner template
  runService =
    {
      name,
      title,
      service,
      logPattern,
      logPath ? "/tmp",
    }:
    ''
      log "Starting ${title} maintenance..."
      systemctl start ${service} || true
      ${statusVarName name}=$?
      log "${title} completed with status $${statusVarName name}"

      SERVICE_LOG=$(${findLatestLog logPattern logPath})
      if [ -n "$SERVICE_LOG" ]; then
        log "Appending ${title} log: $SERVICE_LOG"
        echo -e "\n\n===== ${title} LOG ($(basename "$SERVICE_LOG")) =====\n" >> "$LOG_FILE"
        cat "$SERVICE_LOG" >> "$LOG_FILE"
        
        # Add SnapRAID-specific summary
        if [ "${name}" = "snapraid" ]; then
          echo -e "\n=== SnapRAID Summary ===" >> "$LOG_FILE"
          grep -E '(Scrub|Sync|Diff|smart)' "$SERVICE_LOG" | tail -n 10 >> "$LOG_FILE"
        fi
      fi
    '';

  # Build the service execution script
  serviceExecution = lib.concatMapStrings runService backupServices;

  # Generate status summary lines
  statusSummaryLines = lib.concatMapStringsSep "\n" (
    s:
    let
      varName = statusVarName s.name;
    in
    "- **${s.title}:** \$([ \$${varName} -eq 0 ] && echo '✅ Success' || echo '❌ Failed') (Exit: \$${varName})"
  ) backupServices;

  # Notification logic with cleaner formatting
  notificationLogic =
    let
      statusVars = map (s: statusVarName s.name) backupServices;
      statusChecks = lib.concatMapStringsSep "\n" (var: "[ \$${var} -eq 0 ] && ") statusVars;
    in
    ''
      # Calculate overall status
      OVERALL_STATUS=0
      ${lib.concatMapStringsSep "\n" (var: "if [ \$${var} -ne 0 ]; then OVERALL_STATUS=1; fi") statusVars}

      TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")
      HOSTNAME=$(hostname)

      SUMMARY=$(cat << EOF
      # Backup Chain Complete

      **Host:** $HOSTNAME  
      **Timestamp:** $TIMESTAMP  
      **Overall Status:** $([ $OVERALL_STATUS -eq 0 ] && echo '✅ Success' || echo '⚠️ Failure')

      ## Service Status:
      ${statusSummaryLines}

      **Log Path:** $LOG_FILE
      EOF)

      if [ $OVERALL_STATUS -eq 0 ]; then
        ${notify "✅ Backup Success" "$SUMMARY" "$LOG_FILE"}
      else
        ${notify "⚠️ Backup Issues" "$SUMMARY" "$LOG_FILE"}
      fi

      exit $OVERALL_STATUS
    '';

in
{
  imports = lib.custom.scanPaths ./.;

  systemd.services.backup-chain = {
    description = "Orchestrated Backup Chain";
    path = with pkgs; [
      apprise
      coreutils
      findutils
      gawk
      gnugrep
      hostname
      systemd
      util-linux
    ];

    serviceConfig = {
      Type = "oneshot";
      Nice = 19;
      IOSchedulingClass = "idle";
      CPUSchedulingPolicy = "idle";
    };

    script = ''
      ${scriptPrelude}
      log "Initializing backup chain on $(hostname)"

      ${serviceExecution}

      log "Finalizing backup chain"
      ${notificationLogic}
    '';
  };

  systemd.timers.backup-chain = {
    wantedBy = [ "timers.target" ];
    timerConfig = {
      OnCalendar = "*-*-* 03:00:00";
      Persistent = true;
      RandomizedDelaySec = "5min";
    };
  };

  environment.systemPackages = [ pkgs.apprise ];
  systemd.tmpfiles.rules = [ "d ${logDir} 0755 root root -" ];
}