404 lines
12 KiB
Nix
404 lines
12 KiB
Nix
{
|
||
config,
|
||
lib,
|
||
pkgs,
|
||
...
|
||
}:
|
||
with lib;
|
||
let
|
||
cfg = config.services.backup;
|
||
|
||
# Helper functions
|
||
safeName = name: replaceStrings [ "-" ] [ "_" ] name;
|
||
statusVar = name: "STATUS_${safeName name}";
|
||
findLatestLog = pattern: path: ''
|
||
find "${path}" -name "${pattern}" -type f -printf "%T@ %p\\n" 2>/dev/null \
|
||
| sort -nr | head -1 | cut -d' ' -f2
|
||
'';
|
||
|
||
# Borg service generator
|
||
mkBorgService =
|
||
job:
|
||
let
|
||
borgCommon = ''
|
||
export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes
|
||
export BORG_NON_INTERACTIVE=yes
|
||
'';
|
||
|
||
initRepo = ''
|
||
if [ ! -d "${job.repo}" ]; then
|
||
mkdir -p "${job.repo}"
|
||
${pkgs.borgbackup}/bin/borg init --encryption=none "${job.repo}"
|
||
fi
|
||
'';
|
||
|
||
extractStats = ''
|
||
{
|
||
echo -e "\n==== BACKUP SUMMARY ====\n"
|
||
grep -A10 "Archive name:" $LOG_FILE || echo "No archive stats found"
|
||
echo -e "\n=== Compression ===\n"
|
||
grep "Compressed size:" $LOG_FILE || echo "No compression stats found"
|
||
echo -e "\n=== Duration ===\n"
|
||
grep "Duration:" $LOG_FILE || echo "No duration stats found"
|
||
grep "Throughput:" $LOG_FILE || echo "No throughput stats found"
|
||
echo -e "\n=== Repository ===\n"
|
||
${pkgs.borgbackup}/bin/borg info ${job.repo} --last 1 2>/dev/null || echo "Could not get repository info"
|
||
echo -e "\n=== Storage Space ===\n"
|
||
df -h ${job.repo} | grep -v "Filesystem" || echo "Could not get storage info"
|
||
} > $LOG_FILE.stats
|
||
STATS=$(cat $LOG_FILE.stats || echo "No stats available")
|
||
'';
|
||
|
||
excludeArgs = concatMapStringsSep " " (pattern: "--exclude '${pattern}'") job.excludePatterns;
|
||
in
|
||
{
|
||
services."backup-${job.name}" = {
|
||
description = "Backup ${job.title} with Borg";
|
||
path = with pkgs; [
|
||
borgbackup
|
||
coreutils
|
||
gnugrep
|
||
hostname
|
||
util-linux
|
||
gawk
|
||
];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
IOSchedulingClass = "idle";
|
||
CPUSchedulingPolicy = "idle";
|
||
Nice = 19;
|
||
};
|
||
|
||
script = ''
|
||
${borgCommon}
|
||
${optionalString job.verbose "set -x"}
|
||
|
||
LOG_FILE="/tmp/borg-${job.name}-backup-$(date +%Y%m%d-%H%M%S).log"
|
||
${initRepo}
|
||
|
||
echo "Starting ${job.title} backup at $(date)" > $LOG_FILE
|
||
START_TIME=$(date +%s)
|
||
|
||
${pkgs.borgbackup}/bin/borg create \
|
||
--stats \
|
||
--compression ${job.compression} \
|
||
${excludeArgs} \
|
||
${job.repo}::${job.name}-$(date +%Y-%m-%d_%H%M%S) \
|
||
${job.sourcePath} >> $LOG_FILE 2>&1
|
||
|
||
BACKUP_STATUS=$?
|
||
echo "Total time: $(($(date +%s) - START_TIME)) seconds" >> $LOG_FILE
|
||
${extractStats}
|
||
|
||
echo -e "\nPruning old backups..." >> $LOG_FILE
|
||
${pkgs.borgbackup}/bin/borg prune \
|
||
--keep-daily ${toString job.keepDaily} \
|
||
--keep-weekly ${toString job.keepWeekly} \
|
||
--keep-monthly ${toString job.keepMonthly} \
|
||
${job.repo} >> $LOG_FILE 2>&1
|
||
|
||
PRUNE_STATUS=$?
|
||
|
||
rm -f $LOG_FILE.stats
|
||
exit $BACKUP_STATUS
|
||
'';
|
||
};
|
||
}
|
||
// optionalAttrs (job.schedule != null) {
|
||
timers."backup-${job.name}" = {
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = job.schedule;
|
||
Persistent = true;
|
||
RandomizedDelaySec = "5min";
|
||
};
|
||
};
|
||
};
|
||
|
||
# Orchestration service
|
||
mkChainService =
|
||
let
|
||
allJobs = cfg.jobs ++ cfg.maintenanceJobs;
|
||
initVars = concatMapStringsSep "\n" (j: "${statusVar j.name}=1") allJobs;
|
||
|
||
runJob = job: ''
|
||
# Determine log pattern
|
||
LOG_PATTERN="${if job.logPattern != "" then job.logPattern else "borg-${job.name}-backup-*.log"}"
|
||
|
||
log "Starting ${job.title}..."
|
||
|
||
systemctl start ${if job ? service then job.service else "backup-${job.name}"} || true
|
||
|
||
# Wait for service to complete and get its exit status
|
||
while systemctl is-active ${
|
||
if job ? service then job.service else "backup-${job.name}"
|
||
} >/dev/null 2>&1; do
|
||
sleep 5
|
||
done
|
||
|
||
# Get the actual service exit status
|
||
${statusVar job.name}=$(systemctl show ${
|
||
if job ? service then job.service else "backup-${job.name}"
|
||
} --property=ExecMainStatus --value)
|
||
log "${job.title} completed with status $${statusVar job.name}"
|
||
|
||
# Give logs time to be written
|
||
sleep 2
|
||
|
||
SERVICE_LOG=$(${findLatestLog "$LOG_PATTERN" "${job.logPath}"})
|
||
if [ -n "$SERVICE_LOG" ] && [ -r "$SERVICE_LOG" ]; then
|
||
log "Appending ${job.title} log: $SERVICE_LOG"
|
||
echo -e "\n\n===== ${job.title} LOG =====\n" >> "$LOG_FILE"
|
||
cat "$SERVICE_LOG" >> "$LOG_FILE" 2>/dev/null || echo "Could not read log file" >> "$LOG_FILE"
|
||
else
|
||
log "No readable log found for ${job.title} (pattern: $LOG_PATTERN in ${job.logPath})"
|
||
fi
|
||
'';
|
||
in
|
||
{
|
||
services.backup-chain = {
|
||
description = "Backup Orchestration Chain";
|
||
path = with pkgs; [
|
||
apprise
|
||
coreutils
|
||
findutils
|
||
gnugrep
|
||
hostname
|
||
systemd
|
||
util-linux
|
||
];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
Nice = 19;
|
||
IOSchedulingClass = "idle";
|
||
CPUSchedulingPolicy = "idle";
|
||
};
|
||
|
||
script = ''
|
||
set -uo pipefail
|
||
LOG_FILE="${cfg.logDir}/backup-chain-$(date +%Y%m%d-%H%M%S).log"
|
||
mkdir -p "${cfg.logDir}"
|
||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||
|
||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"; }
|
||
|
||
# Initialize status tracking
|
||
declare -A JOB_STATUS
|
||
${concatMapStrings (j: "JOB_STATUS['${j.name}']=1\n") allJobs}
|
||
|
||
log "Starting backup chain on $(hostname)"
|
||
|
||
${concatMapStrings (job: ''
|
||
log "Starting ${job.title}..."
|
||
systemctl start ${if job ? service then job.service else "backup-${job.name}"} || true
|
||
|
||
# Wait for completion
|
||
while systemctl is-active ${
|
||
if job ? service then job.service else "backup-${job.name}"
|
||
} >/dev/null 2>&1; do
|
||
sleep 5
|
||
done
|
||
|
||
# Capture exit status
|
||
JOB_STATUS['${job.name}']=$(systemctl show ${
|
||
if job ? service then job.service else "backup-${job.name}"
|
||
} --property=ExecMainStatus --value)
|
||
log "${job.title} completed with status ''${JOB_STATUS['${job.name}']}"
|
||
|
||
# Give logs time to be written
|
||
sleep 2
|
||
|
||
# Find and append logs
|
||
LOG_PATTERN="${if job.logPattern != "" then job.logPattern else "borg-${job.name}-backup-*.log"}"
|
||
SERVICE_LOG=$(${findLatestLog "$LOG_PATTERN" "${job.logPath}"})
|
||
if [ -n "$SERVICE_LOG" ] && [ -r "$SERVICE_LOG" ]; then
|
||
log "Appending ${job.title} log: $SERVICE_LOG"
|
||
echo -e "\n\n===== ${job.title} LOG =====\n" >> "$LOG_FILE"
|
||
cat "$SERVICE_LOG" >> "$LOG_FILE" 2>/dev/null || echo "Could not read log file" >> "$LOG_FILE"
|
||
else
|
||
log "No readable log found for ${job.title} (pattern: $LOG_PATTERN in ${job.logPath})"
|
||
fi
|
||
'') allJobs}
|
||
|
||
# Calculate overall status
|
||
OVERALL_STATUS=0
|
||
for status in "''${JOB_STATUS[@]}"; do
|
||
if [ "$status" -ne 0 ]; then
|
||
OVERALL_STATUS=1
|
||
break
|
||
fi
|
||
done
|
||
|
||
# Build summary
|
||
HOSTNAME=$(hostname)
|
||
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
# Build job status lines
|
||
JOB_LINES=""
|
||
${concatMapStrings (job: ''
|
||
if [ ''${JOB_STATUS['${job.name}']} -eq 0 ]; then
|
||
STATUS_ICON="✅"
|
||
else
|
||
STATUS_ICON="❌"
|
||
fi
|
||
JOB_LINES="$JOB_LINES- **${job.title}:** $STATUS_ICON (Exit: ''${JOB_STATUS['${job.name}']})
|
||
"
|
||
'') allJobs}
|
||
|
||
# Create the final summary
|
||
SUMMARY="# Backup Chain Complete
|
||
|
||
**Host:** $HOSTNAME
|
||
**Timestamp:** $TIMESTAMP
|
||
**Overall Status:** $([ $OVERALL_STATUS -eq 0 ] && echo '✅ Success' || echo '⚠️ Failure')
|
||
|
||
## Job Status:
|
||
$JOB_LINES
|
||
**Log Path:** $LOG_FILE"
|
||
|
||
# Send notification
|
||
${pkgs.apprise}/bin/apprise -vv -i "markdown" \
|
||
-t "Backup $([ $OVERALL_STATUS -eq 0 ] && echo '✅' || echo '⚠️')" \
|
||
-b "$SUMMARY" \
|
||
--attach "file://$LOG_FILE" \
|
||
"${cfg.notificationUrl}" || true
|
||
|
||
exit $OVERALL_STATUS
|
||
'';
|
||
};
|
||
|
||
timers.backup-chain = mkIf cfg.enableChainTimer {
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = cfg.chainSchedule;
|
||
Persistent = true;
|
||
RandomizedDelaySec = "5min";
|
||
};
|
||
};
|
||
};
|
||
|
||
# Job types
|
||
borgJobType = types.submodule {
|
||
options = {
|
||
name = mkOption {
|
||
type = types.str;
|
||
description = "Unique service name";
|
||
};
|
||
title = mkOption {
|
||
type = types.str;
|
||
description = "Human-readable title";
|
||
};
|
||
repo = mkOption {
|
||
type = types.str;
|
||
description = "Borg repository path";
|
||
};
|
||
sourcePath = mkOption {
|
||
type = types.str;
|
||
description = "Path to back up";
|
||
};
|
||
schedule = mkOption {
|
||
type = types.nullOr types.str;
|
||
default = null;
|
||
description = "Timer schedule for standalone execution";
|
||
};
|
||
keepDaily = mkOption {
|
||
type = types.int;
|
||
default = 7;
|
||
};
|
||
keepWeekly = mkOption {
|
||
type = types.int;
|
||
default = 4;
|
||
};
|
||
keepMonthly = mkOption {
|
||
type = types.int;
|
||
default = 3;
|
||
};
|
||
verbose = mkOption {
|
||
type = types.bool;
|
||
default = false;
|
||
};
|
||
compression = mkOption {
|
||
type = types.str;
|
||
default = "zstd,15";
|
||
};
|
||
excludePatterns = mkOption {
|
||
type = types.listOf types.str;
|
||
default = [
|
||
"*.tmp"
|
||
"*/tmp/*"
|
||
];
|
||
};
|
||
logPattern = mkOption {
|
||
type = types.str;
|
||
default = "";
|
||
description = "Log pattern for chain to find";
|
||
};
|
||
logPath = mkOption {
|
||
type = types.str;
|
||
default = "/tmp";
|
||
description = "Path to search for logs";
|
||
};
|
||
};
|
||
};
|
||
|
||
maintenanceJobType = types.submodule {
|
||
options = {
|
||
name = mkOption { type = types.str; };
|
||
title = mkOption { type = types.str; };
|
||
service = mkOption {
|
||
type = types.str;
|
||
description = "Systemd service to run";
|
||
};
|
||
logPattern = mkOption { type = types.str; };
|
||
logPath = mkOption {
|
||
type = types.str;
|
||
default = "/tmp";
|
||
};
|
||
};
|
||
};
|
||
|
||
in
|
||
{
|
||
options.services.backup = {
|
||
enable = mkEnableOption "backup system";
|
||
notificationUrl = mkOption { type = types.str; };
|
||
logDir = mkOption {
|
||
type = types.str;
|
||
default = "/var/log/backups";
|
||
};
|
||
jobs = mkOption {
|
||
type = types.listOf borgJobType;
|
||
default = [ ];
|
||
};
|
||
maintenanceJobs = mkOption {
|
||
type = types.listOf maintenanceJobType;
|
||
default = [ ];
|
||
};
|
||
enableChainTimer = mkOption {
|
||
type = types.bool;
|
||
default = false;
|
||
};
|
||
chainSchedule = mkOption {
|
||
type = types.str;
|
||
default = "*-*-* 03:00:00";
|
||
};
|
||
};
|
||
|
||
config = mkIf cfg.enable {
|
||
environment.systemPackages = with pkgs; [
|
||
borgbackup
|
||
apprise
|
||
];
|
||
|
||
systemd = mkMerge [
|
||
# Create log directory
|
||
{ tmpfiles.rules = [ "d ${cfg.logDir} 0755 root root -" ]; }
|
||
|
||
# Individual backup services
|
||
(mkMerge (map mkBorgService cfg.jobs))
|
||
|
||
# Backup chain orchestration
|
||
(mkIf (cfg.jobs != [ ] || cfg.maintenanceJobs != [ ]) mkChainService)
|
||
];
|
||
};
|
||
}
|