diff --git a/hosts/nixos/sock/config/backups/borg.nix b/hosts/nixos/sock/config/backups/borg.nix new file mode 100644 index 0000000..a9edcb2 --- /dev/null +++ b/hosts/nixos/sock/config/backups/borg.nix @@ -0,0 +1,193 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # Common repositories + ochreStorageRepo = "/pool/Backups/OchreStorage"; + + # Shared environment setup + borgCommonSettings = '' + # Don't use cache to avoid issues with concurrent backups + export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes + export BORG_NON_INTERACTIVE=yes + ''; + + # Common packages needed for backups + commonBorgPath = with pkgs; [ + borgbackup + coreutils + apprise + gnugrep + hostname + util-linux + gawk + ]; + + # Repository initialization + initRepo = repo: '' + if [ ! -d "${repo}" ]; then + mkdir -p "${repo}" + ${pkgs.borgbackup}/bin/borg init --encryption=none "${repo}" + fi + ''; + + # Notification system + apprise-url = config.secretsSpec.users.admin.smtp.notifyUrl; + sendNotification = title: message: '' + ${pkgs.apprise}/bin/apprise -t "${title}" -b "${message}" "${apprise-url}" || true + ''; + + # Statistics generation + extractBorgStats = logFile: repoPath: '' + { + echo -e "\n==== BACKUP SUMMARY ====\n" + grep -A10 "Archive name:" ${logFile} || echo "No archive stats found" + echo -e "\n=== Compression ===\n" + grep "Compressed size:" ${logFile} || echo "No compression stats found" + echo -e "\n=== Duration ===\n" + grep "Duration:" ${logFile} || echo "No duration stats found" + grep "Throughput:" ${logFile} || echo "No throughput stats found" + echo -e "\n=== Repository ===\n" + ${pkgs.borgbackup}/bin/borg info ${repoPath} --last 1 2>/dev/null || echo "Could not get repository info" + echo -e "\n=== Storage Space ===\n" + df -h ${repoPath} | grep -v "Filesystem" || echo "Could not get storage info" + } > ${logFile}.stats + STATS=$(cat ${logFile}.stats || echo "No stats available") + ''; + + # Unified backup service generator with optional features + mkBorgBackupService = + { + name, + title, + repo, + sourcePath, + keepDaily, + keepWeekly, + keepMonthly, + schedule ? null, + enableNotifications ? true, + verbose ? false, + }: + let + maybeCreateTimer = lib.optionalAttrs (schedule != null) { + timers."backup-${name}" = { + description = "Timer for ${title} Backup"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = schedule; + Persistent = true; + RandomizedDelaySec = "5min"; + }; + }; + }; + + logPrefix = if verbose then "set -x;" else ""; + in + { + services."backup-${name}" = { + description = "Backup ${title} with Borg"; + inherit (commonServiceConfig) path serviceConfig; + + script = '' + ${borgCommonSettings} + ${logPrefix} # Add verbose logging if enabled + + LOG_FILE="/tmp/borg-${name}-backup-$(date +%Y%m%d-%H%M%S).log" + ${initRepo repo} + + echo "Starting ${title} backup at $(date)" > $LOG_FILE + ARCHIVE_NAME="${name}-$(date +%Y-%m-%d_%H%M%S)" + START_TIME=$(date +%s) + + # Add verbose output redirection if enabled + ${if verbose then "exec 3>&1 4>&2" else ""} + ${pkgs.borgbackup}/bin/borg create \ + --stats \ + --compression zstd,15 \ + --exclude '*.tmp' \ + --exclude '*/tmp/*' \ + ${repo}::$ARCHIVE_NAME \ + ${sourcePath} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""} + + BACKUP_STATUS=$? + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + echo "Total time: $DURATION seconds ($(date -d@$DURATION -u +%H:%M:%S))" >> $LOG_FILE + + ${extractBorgStats "$LOG_FILE" "${repo}"} + + echo -e "\nPruning old backups..." >> $LOG_FILE + ${pkgs.borgbackup}/bin/borg prune \ + --keep-daily ${toString keepDaily} \ + --keep-weekly ${toString keepWeekly} \ + --keep-monthly ${toString keepMonthly} \ + ${repo} >> $LOG_FILE 2>&1 ${if verbose then "| tee /dev/fd/3" else ""} + + PRUNE_STATUS=$? + + echo -e "\nRemaining archives after pruning:" >> $LOG_FILE + ${pkgs.borgbackup}/bin/borg list ${repo} >> $LOG_FILE 2>&1 || true + + ${ + if enableNotifications then + '' + if [ $BACKUP_STATUS -eq 0 ] && [ $PRUNE_STATUS -eq 0 ]; then + ${sendNotification "✅ ${title} Backup Complete" "${title} backup completed successfully on $(hostname) at $(date)\nDuration: $(date -d@$DURATION -u +%H:%M:%S)\n\n$STATS"} + else + ${sendNotification "❌ ${title} Backup Failed" "${title} backup failed on $(hostname) at $(date)\n\nBackup Status: $BACKUP_STATUS\nPrune Status: $PRUNE_STATUS\n\nPartial Stats:\n$STATS\n\nSee $LOG_FILE for details"} + fi + '' + else + "echo 'Notifications disabled' >> $LOG_FILE" + } + + rm -f $LOG_FILE.stats + exit $BACKUP_STATUS + ''; + }; + + } + // maybeCreateTimer; + + # Common service configuration + commonServiceConfig = { + path = commonBorgPath; + serviceConfig = { + Type = "oneshot"; + IOSchedulingClass = "idle"; + CPUSchedulingPolicy = "idle"; + Nice = 19; + }; + }; + +in +{ + environment.systemPackages = with pkgs; [ + borgbackup + ]; + + systemd = lib.mkMerge [ + (mkBorgBackupService { + name = "ochre-storage"; + title = "Ochre Storage"; + repo = ochreStorageRepo; + sourcePath = "/OchreStorage"; + # INFO: This shit confusing but basically + # keeps the last 7 days, + # then keeps AT LEAST ONE for last 4 weeks + # and finally AT LEAST ONE for the last 3 months + keepDaily = 7; + keepWeekly = 4; + keepMonthly = 3; + # No schedule = no timer created + # schedule = "*-*-* 03:00:00"; + enableNotifications = false; + verbose = true; + }) + ]; +} diff --git a/hosts/nixos/sock/config/backups/default.nix b/hosts/nixos/sock/config/backups/default.nix new file mode 100644 index 0000000..4e72634 --- /dev/null +++ b/hosts/nixos/sock/config/backups/default.nix @@ -0,0 +1,184 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # Shared configuration + logDir = "/var/log/backups"; + backupServices = [ + { + name = "ochre_storage"; + title = "Ochre Storage"; + service = "backup-ochre-storage.service"; + logPattern = "borg-ochre-storage-backup-*.log"; + } + ]; + + # Helper functions + users = config.secretsSpec.users; + notify = + title: message: logFile: + let + attachArg = if logFile == "" then "" else "--attach \"file://${logFile}\""; + appriseUrl = lib.custom.mkAppriseUrl users.admin.smtp "relay@ryot.foo"; + in + '' + ${pkgs.apprise}/bin/apprise -vv -i "markdown" -t "${title}" \ + -b "${message}" \ + ${attachArg} \ + "${appriseUrl}" || true + ''; + + findLatestLog = pattern: path: '' + find "${path}" -name "${pattern}" -type f -printf "%T@ %p\\n" 2>/dev/null \ + | sort -nr | head -1 | cut -d' ' -f2 + ''; + + # Generate safe variable name (replace hyphens with underscores) + safeName = name: lib.replaceStrings [ "-" ] [ "_" ] name; + + # Generate status variable references + statusVarName = name: "STATUS_${safeName name}"; + + # Common script utilities + scriptPrelude = '' + set -uo pipefail + LOG_FILE="${logDir}/backup-chain-$(date +%Y%m%d-%H%M%S).log" + mkdir -p "${logDir}" + exec > >(tee -a "$LOG_FILE") 2>&1 + + log() { + echo "[$(date "+%Y-%m-%d %H:%M:%S")] $1" + } + + # Initialize all status variables + ${lib.concatMapStringsSep "\n" (s: "${statusVarName s.name}=1") backupServices} + ''; + + # Service runner template + runService = + { + name, + title, + service, + logPattern, + logPath ? "/tmp", + }: + '' + log "Starting ${title} maintenance..." + systemctl start ${service} || true + ${statusVarName name}=$? + log "${title} completed with status $${statusVarName name}" + + SERVICE_LOG=$(${findLatestLog logPattern logPath}) + if [ -n "$SERVICE_LOG" ]; then + log "Appending ${title} log: $SERVICE_LOG" + echo -e "\n\n===== ${title} LOG ($(basename "$SERVICE_LOG")) =====\n" >> "$LOG_FILE" + cat "$SERVICE_LOG" >> "$LOG_FILE" + + # Add SnapRAID-specific summary + if [ "${name}" = "snapraid" ]; then + echo -e "\n=== SnapRAID Summary ===" >> "$LOG_FILE" + grep -E '(Scrub|Sync|Diff|smart)' "$SERVICE_LOG" | tail -n 10 >> "$LOG_FILE" + fi + fi + ''; + + # Build the service execution script + serviceExecution = lib.concatMapStrings runService backupServices; + + # Generate status summary lines + statusSummaryLines = lib.concatMapStringsSep "\n" ( + s: + let + varName = statusVarName s.name; + in + "- **${s.title}:** \$([ \$${varName} -eq 0 ] && echo '✅ Success' || echo '❌ Failed') (Exit: \$${varName})" + ) backupServices; + + # Notification logic with cleaner formatting + notificationLogic = + let + statusVars = map (s: statusVarName s.name) backupServices; + statusChecks = lib.concatMapStringsSep "\n" (var: "[ \$${var} -eq 0 ] && ") statusVars; + in + '' + # Calculate overall status + OVERALL_STATUS=0 + ${lib.concatMapStringsSep "\n" (var: "if [ \$${var} -ne 0 ]; then OVERALL_STATUS=1; fi") statusVars} + + TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S") + HOSTNAME=$(hostname) + + SUMMARY=$(cat << EOF + # Backup Chain Complete + + **Host:** $HOSTNAME + **Timestamp:** $TIMESTAMP + **Overall Status:** $([ $OVERALL_STATUS -eq 0 ] && echo '✅ Success' || echo '⚠️ Failure') + + ## Service Status: + ${statusSummaryLines} + + **Log Path:** $LOG_FILE + EOF) + + if [ $OVERALL_STATUS -eq 0 ]; then + ${notify "✅ Backup Success" "$SUMMARY" "$LOG_FILE"} + else + ${notify "⚠️ Backup Issues" "$SUMMARY" "$LOG_FILE"} + fi + + exit $OVERALL_STATUS + ''; + +in +{ + imports = lib.custom.scanPaths ./.; + + systemd.services.backup-chain = { + description = "Orchestrated Backup Chain"; + path = with pkgs; [ + apprise + coreutils + findutils + gawk + gnugrep + hostname + systemd + util-linux + ]; + + serviceConfig = { + Type = "oneshot"; + Nice = 19; + IOSchedulingClass = "idle"; + CPUSchedulingPolicy = "idle"; + }; + + script = '' + ${scriptPrelude} + log "Initializing backup chain on $(hostname)" + + ${serviceExecution} + + log "Finalizing backup chain" + ${notificationLogic} + ''; + }; + + systemd.timers.backup-chain = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*-*-* 03:00:00"; + Persistent = true; + RandomizedDelaySec = "5min"; + }; + }; + + environment.systemPackages = [ pkgs.apprise ]; + systemd.tmpfiles.rules = [ "d ${logDir} 0755 root root -" ]; +} diff --git a/hosts/nixos/sock/config/default.nix b/hosts/nixos/sock/config/default.nix new file mode 100644 index 0000000..b23287b --- /dev/null +++ b/hosts/nixos/sock/config/default.nix @@ -0,0 +1,4 @@ +{ lib, ... }: +{ + imports = lib.custom.scanPaths ./.; +} diff --git a/hosts/nixos/sock/config/komodo/default.nix b/hosts/nixos/sock/config/komodo/default.nix new file mode 100644 index 0000000..196a34f --- /dev/null +++ b/hosts/nixos/sock/config/komodo/default.nix @@ -0,0 +1,183 @@ +# Auto-generated using compose2nix v0.3.1. +{ + config, + lib, + pkgs, + ... +}: +let + # Only available in the Sock LXC + DockerStorage = "/OchreStorage/komodo"; + env = config.secretsSpec.docker.komodo-sock; +in +{ + # Containers + virtualisation.oci-containers.containers."komodo-core" = { + image = "ghcr.io/moghtech/komodo-core:latest"; + environment = env; + volumes = [ + "${DockerStorage}/cache:/repo-cache:rw" + ]; + ports = [ + "9120:9120/tcp" + ]; + labels = { + "komodo.skip" = ""; + }; + dependsOn = [ + "komodo-mongo" + ]; + log-driver = "local"; + extraOptions = [ + "--network-alias=core" + "--network=komodo_default" + "--pull=always" + ]; + }; + + systemd.services."docker-komodo-core" = { + serviceConfig = { + Restart = lib.mkOverride 90 "always"; + RestartMaxDelaySec = lib.mkOverride 90 "1m"; + RestartSec = lib.mkOverride 90 "100ms"; + RestartSteps = lib.mkOverride 90 9; + }; + after = [ + "docker-network-komodo_default.service" + # "docker-volume-komodo_repo-cache.service" + ]; + requires = [ + "docker-network-komodo_default.service" + # "docker-volume-komodo_repo-cache.service" + ]; + partOf = [ + "docker-compose-komodo-root.target" + ]; + wantedBy = [ + "docker-compose-komodo-root.target" + ]; + }; + + virtualisation.oci-containers.containers."komodo-mongo" = { + image = "mongo"; + environment = env; + volumes = [ + "${DockerStorage}/mongo/config:/data/configdb:rw" + "${DockerStorage}/mongo/data:/data/db:rw" + ]; + cmd = [ + "--quiet" + "--wiredTigerCacheSizeGB" + "0.25" + ]; + labels = { + "komodo.skip" = ""; + }; + log-driver = "local"; + extraOptions = [ + "--network-alias=mongo" + "--network=komodo_default" + ]; + }; + + systemd.services."docker-komodo-mongo" = { + serviceConfig = { + Restart = lib.mkOverride 90 "always"; + RestartMaxDelaySec = lib.mkOverride 90 "1m"; + RestartSec = lib.mkOverride 90 "100ms"; + RestartSteps = lib.mkOverride 90 9; + }; + after = [ + "docker-network-komodo_default.service" + # "docker-volume-komodo_mongo-config.service" + # "docker-volume-komodo_mongo-data.service" + ]; + requires = [ + "docker-network-komodo_default.service" + # "docker-volume-komodo_mongo-config.service" + # "docker-volume-komodo_mongo-data.service" + ]; + partOf = [ + "docker-compose-komodo-root.target" + ]; + wantedBy = [ + "docker-compose-komodo-root.target" + ]; + }; + + virtualisation.oci-containers.containers."komodo-periphery" = { + image = "ghcr.io/moghtech/komodo-periphery:latest"; + environment = env; + volumes = [ + "/proc:/proc:rw" + "/var/run/docker.sock:/var/run/docker.sock:rw" + "${DockerStorage}/repos:/etc/komodo/repos:rw" + "${DockerStorage}/ssl:/etc/komodo/ssl:rw" + "${DockerStorage}/stacks:${DockerStorage}/stacks:rw" + ]; + ports = [ + "8120:8120/tcp" + ]; + labels = { + "komodo.skip" = ""; + }; + log-driver = "local"; + extraOptions = [ + "--network-alias=periphery" + "--network=komodo_default" + "--pull=always" + ]; + }; + + systemd.services."docker-komodo-periphery" = { + serviceConfig = { + Restart = lib.mkOverride 90 "always"; + RestartMaxDelaySec = lib.mkOverride 90 "1m"; + RestartSec = lib.mkOverride 90 "100ms"; + RestartSteps = lib.mkOverride 90 9; + }; + after = [ + "docker-network-komodo_default.service" + # "docker-volume-komodo_repos.service" + # "docker-volume-komodo_ssl-certs.service" + # "docker-volume-komodo_stacks.service" + ]; + requires = [ + "docker-network-komodo_default.service" + # "docker-volume-komodo_repos.service" + # "docker-volume-komodo_ssl-certs.service" + # "docker-volume-komodo_stacks.service" + ]; + partOf = [ + "docker-compose-komodo-root.target" + ]; + wantedBy = [ + "docker-compose-komodo-root.target" + ]; + }; + + # Networks + systemd.services."docker-network-komodo_default" = { + path = [ pkgs.docker ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStop = "docker network rm -f komodo_default"; + }; + script = '' + docker network inspect komodo_default || docker network create komodo_default + ''; + partOf = [ "docker-compose-komodo-root.target" ]; + wantedBy = [ "docker-compose-komodo-root.target" ]; + }; + + # Root service + # When started, this will automatically create all resources and start + # the containers. When stopped, this will teardown all resources. + systemd.targets."docker-compose-komodo-root" = { + unitConfig = { + Description = "Root target generated by compose2nix."; + }; + wantedBy = [ "multi-user.target" ]; + }; +} diff --git a/hosts/nixos/sock/default.nix b/hosts/nixos/sock/default.nix new file mode 100644 index 0000000..00476fc --- /dev/null +++ b/hosts/nixos/sock/default.nix @@ -0,0 +1,69 @@ +############################################################### +# +# Sock - LXC Container +# NixOS container, Intel N150 (4 Cores), 8GB/2GB RAM/SWAP +# +# Docker Environment, Managed by with Komodo +# +############################################################### + +{ + lib, + config, + pkgs, + ... +}: +let + username = "toph"; + user = config.secretsSpec.users.${username}; + firewall = config.secretsSpec.firewall.sock; +in +{ + imports = lib.flatten [ + ## Sock Only ## + ./config + + ## Hardware ## + ./hardware.nix + + (map lib.custom.relativeToRoot [ + ## Required Configs ## + "hosts/global/core" + + ## Optional Configs ## + "hosts/global/common/acme" + "hosts/global/common/docker.nix" + ]) + ]; + + ## Host Specifications ## + hostSpec = { + hostName = "sock"; + username = username; + hashedPassword = user.hashedPassword; + email = user.email; + handle = user.handle; + userFullName = user.fullName; + isServer = true; + isMinimal = true; + }; + + networking = { + enableIPv6 = false; + firewall = { + allowedTCPPorts = firewall.allowedTCPPorts; + allowedTCPPortRanges = firewall.allowedTCPPortRanges; + allowedUDPPorts = firewall.allowedUDPPorts; + }; + }; + + ## System-wide packages ## + programs.nix-ld.enable = true; + environment.systemPackages = with pkgs; [ + lazydocker + compose2nix + ]; + + # https://wiki.nixos.org/wiki/FAQ/When_do_I_update_stateVersion + system.stateVersion = "25.05"; +} diff --git a/hosts/nixos/sock/hardware.nix b/hosts/nixos/sock/hardware.nix new file mode 100644 index 0000000..06f2f4a --- /dev/null +++ b/hosts/nixos/sock/hardware.nix @@ -0,0 +1,21 @@ +{ + lib, + config, + ... +}: +let + username = config.hostSpec.username; +in +{ + imports = lib.flatten [ + (map lib.custom.relativeToRoot [ + "hosts/global/common/system/lxc.nix" + "hosts/global/common/system/pool.nix" + ]) + ]; + + # Ochre has no access to PVE DockerStorage, so sock will have its own storage + systemd.user.tmpfiles.rules = [ + "d /OchreStorage 2775 ${username} ryot -" + ]; +} diff --git a/secrets.nix b/secrets.nix index 07519bf..5bddb71 100644 Binary files a/secrets.nix and b/secrets.nix differ