From 9a28ac9e4490cf39d781171a52197babc121b2cc Mon Sep 17 00:00:00 2001 From: Chris Toph Date: Wed, 4 Sep 2024 00:08:14 -0400 Subject: [PATCH] Custom pkg for snapraid-runner needs more configuration once snapraid itself is setup --- flake.nix | 5 +- nixos/configuration.nix | 11 +- nixos/imports/snapraid-runner.nix | 5 + nixos/overlays/default.nix | 7 + nixos/pkgs/snapraid-runner/default.nix | 56 ++++ .../pkgs/snapraid-runner/snapraid-runner.conf | 44 +++ nixos/pkgs/snapraid-runner/snapraid-runner.py | 314 ++++++++++++++++++ 7 files changed, 439 insertions(+), 3 deletions(-) create mode 100644 nixos/imports/snapraid-runner.nix create mode 100644 nixos/overlays/default.nix create mode 100644 nixos/pkgs/snapraid-runner/default.nix create mode 100644 nixos/pkgs/snapraid-runner/snapraid-runner.conf create mode 100644 nixos/pkgs/snapraid-runner/snapraid-runner.py diff --git a/flake.nix b/flake.nix index 0084a64..f258c3d 100644 --- a/flake.nix +++ b/flake.nix @@ -13,7 +13,10 @@ let system = "x86_64-linux"; lib = nixpkgs.lib; - pkgs = nixpkgs.legacyPackages.${system}; + pkgs = import nixpkgs { + inherit system; + overlays = [ (import ./overlays) ]; + }; in { nixosConfigurations = { cloud = lib.nixosSystem { diff --git a/nixos/configuration.nix b/nixos/configuration.nix index b89c3dc..4cb1320 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -12,11 +12,16 @@ in { "${modulesPath}/virtualisation/lxc-container.nix" # Include the container-specific autogenerated configuration. #./lxd.nix - this has to be commented out from the system tarball + + # Snapraid-runner + ./imports/snapraid-runner.nix # Import hardware configuration. ./hardware-configuration.nix ]; - + + nixpkgs.overlays = [ (import ./overlays) ]; + # NETWORKING networking = { firewall = { @@ -91,7 +96,9 @@ in { openssh ranger sshfs - wget + snapraid + snapraid-runner + wget ]; # PROGRAMS & SERVICES diff --git a/nixos/imports/snapraid-runner.nix b/nixos/imports/snapraid-runner.nix new file mode 100644 index 0000000..13edc0b --- /dev/null +++ b/nixos/imports/snapraid-runner.nix @@ -0,0 +1,5 @@ +{ pkgs, ... }: + +{ + environment.etc."snapraid-runner.conf".text = builtins.readFile ../pkgs/snapraid-runner/snapraid-runner.conf; +} diff --git a/nixos/overlays/default.nix b/nixos/overlays/default.nix new file mode 100644 index 0000000..4bf482e --- /dev/null +++ b/nixos/overlays/default.nix @@ -0,0 +1,7 @@ +self: super: + +let + callPackage = super.callPackage; +in { + snapraid-runner = callPackage ../pkgs/snapraid-runner { }; +} \ No newline at end of file diff --git a/nixos/pkgs/snapraid-runner/default.nix b/nixos/pkgs/snapraid-runner/default.nix new file mode 100644 index 0000000..28208c2 --- /dev/null +++ b/nixos/pkgs/snapraid-runner/default.nix @@ -0,0 +1,56 @@ +{ + stdenv, + lib, + pkgs, + ... +}: + +let + snapraid-runner-py = pkgs.writeTextFile { + name = "snapraid-runner.py"; + executable = true; + destination = "/bin/snapraid-runner.py"; + text = builtins.readFile ./snapraid-runner.py; + }; + + snapraid-runner = pkgs.writeTextFile { + name = "snapraid-runner.sh"; + executable = true; + destination = "/bin/snapraid-runner"; + text = ''#!${pkgs.stdenv.shell} + +# Check if the "-c" option is present +config_option_present=false +for arg in "$@"; do + if [ "$arg" = "-c" ]; then + config_option_present=true + break + fi +done + +# Add the default config option if not present +if [ "$config_option_present" = false ]; then + set -- "-c" "/etc/snapraid-runner.conf" "$@" +fi + +${pkgs.python311}/bin/python3 ${snapraid-runner-py}/bin/snapraid-runner.py "$@" +''; + }; + +in +stdenv.mkDerivation rec { + pname = "snapraid-runner"; + version = "8f78f9f1af8ca5a9b6469a6c142cab2577157331"; + + buildInputs = [ + snapraid-runner + ]; + + builder = pkgs.writeTextFile { + name = "builder.sh"; + text = ''. $stdenv/setup +mkdir -p $out/bin +ln -sf ${snapraid-runner}/bin/snapraid-runner $out/bin/snapraid-runner''; + }; +} + diff --git a/nixos/pkgs/snapraid-runner/snapraid-runner.conf b/nixos/pkgs/snapraid-runner/snapraid-runner.conf new file mode 100644 index 0000000..9b74379 --- /dev/null +++ b/nixos/pkgs/snapraid-runner/snapraid-runner.conf @@ -0,0 +1,44 @@ +[snapraid] +; path to the snapraid executable (e.g. /bin/snapraid) +executable = /usr/bin/snapraid +; path to the snapraid config to be used +config = /etc/snapraid.conf +; abort operation if there are more deletes than this, set to -1 to disable +deletethreshold = 40 +; if you want touch to be ran each time +touch = false + +[logging] +; logfile to write to, leave empty to disable +file = /var/log/snapraid-runner.log +; maximum logfile size in KiB, leave empty for infinite +maxsize = 5000 + +[email] +; when to send an email, comma-separated list of [success, error] +sendon = success,error +; set to false to get full programm output via email +short = true +subject = [SnapRAID] Status Report: +from = cloud@ryot.foo +to = [REDACTED] +; maximum email size in KiB +maxsize = 500 + +[smtp] +host = ryot.foo +; leave empty for default port +port = +; set to "true" to activate +ssl = true +tls = true +user = admin +password = [REDACTED] + +[scrub] +; set to true to run scrub after sync +enabled = true +; scrub plan - either a percentage or one of [bad, new, full] +plan = 12 +; minimum block age (in days) for scrubbing. Only used with percentage plans +older-than = 10 \ No newline at end of file diff --git a/nixos/pkgs/snapraid-runner/snapraid-runner.py b/nixos/pkgs/snapraid-runner/snapraid-runner.py new file mode 100644 index 0000000..09fcf94 --- /dev/null +++ b/nixos/pkgs/snapraid-runner/snapraid-runner.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +import argparse +import configparser +import logging +import logging.handlers +import os.path +import subprocess +import sys +import threading +import time +import traceback +from collections import Counter, defaultdict +from io import StringIO + +# Global variables +config = None +email_log = None + + +def tee_log(infile, out_lines, log_level): + """ + Create a thread that saves all the output on infile to out_lines and + logs every line with log_level + """ + def tee_thread(): + for line in iter(infile.readline, ""): + logging.log(log_level, line.rstrip()) + out_lines.append(line) + infile.close() + t = threading.Thread(target=tee_thread) + t.daemon = True + t.start() + return t + + +def snapraid_command(command, args={}, *, allow_statuscodes=[]): + """ + Run snapraid command + Raises subprocess.CalledProcessError if errorlevel != 0 + """ + arguments = ["--conf", config["snapraid"]["config"], + "--quiet"] + for (k, v) in args.items(): + arguments.extend(["--" + k, str(v)]) + p = subprocess.Popen( + [config["snapraid"]["executable"], command] + arguments, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # Snapraid always outputs utf-8 on windows. On linux, utf-8 + # also seems a sensible assumption. + encoding="utf-8", + errors="replace") + out = [] + threads = [ + tee_log(p.stdout, out, logging.OUTPUT), + tee_log(p.stderr, [], logging.OUTERR)] + for t in threads: + t.join() + ret = p.wait() + # sleep for a while to make pervent output mixup + time.sleep(0.3) + if ret == 0 or ret in allow_statuscodes: + return out + else: + raise subprocess.CalledProcessError(ret, "snapraid " + command) + + +def send_email(success): + import smtplib + from email.mime.text import MIMEText + from email import charset + + if len(config["smtp"]["host"]) == 0: + logging.error("Failed to send email because smtp host is not set") + return + + # use quoted-printable instead of the default base64 + charset.add_charset("utf-8", charset.SHORTEST, charset.QP) + if success: + body = "SnapRAID job completed successfully:\n\n\n" + else: + body = "Error during SnapRAID job:\n\n\n" + + log = email_log.getvalue() + maxsize = config['email'].get('maxsize', 500) * 1024 + if maxsize and len(log) > maxsize: + cut_lines = log.count("\n", maxsize // 2, -maxsize // 2) + log = ( + "NOTE: Log was too big for email and was shortened\n\n" + + log[:maxsize // 2] + + "[...]\n\n\n --- LOG WAS TOO BIG - {} LINES REMOVED --\n\n\n[...]".format( + cut_lines) + + log[-maxsize // 2:]) + body += log + + msg = MIMEText(body, "plain", "utf-8") + msg["Subject"] = config["email"]["subject"] + \ + (" SUCCESS" if success else " ERROR") + msg["From"] = config["email"]["from"] + msg["To"] = config["email"]["to"] + smtp = {"host": config["smtp"]["host"]} + if config["smtp"]["port"]: + smtp["port"] = config["smtp"]["port"] + if config["smtp"]["ssl"]: + server = smtplib.SMTP_SSL(**smtp) + else: + server = smtplib.SMTP(**smtp) + if config["smtp"]["tls"]: + server.starttls() + if config["smtp"]["user"]: + server.login(config["smtp"]["user"], config["smtp"]["password"]) + server.sendmail( + config["email"]["from"], + [config["email"]["to"]], + msg.as_string()) + server.quit() + + +def finish(is_success): + if ("error", "success")[is_success] in config["email"]["sendon"]: + try: + send_email(is_success) + except Exception: + logging.exception("Failed to send email") + if is_success: + logging.info("Run finished successfully") + else: + logging.error("Run failed") + sys.exit(0 if is_success else 1) + + +def load_config(args): + global config + parser = configparser.RawConfigParser() + parser.read(args.conf) + sections = ["snapraid", "logging", "email", "smtp", "scrub"] + config = dict((x, defaultdict(lambda: "")) for x in sections) + for section in parser.sections(): + for (k, v) in parser.items(section): + config[section][k] = v.strip() + + int_options = [ + ("snapraid", "deletethreshold"), ("logging", "maxsize"), + ("scrub", "older-than"), ("email", "maxsize"), + ] + for section, option in int_options: + try: + config[section][option] = int(config[section][option]) + except ValueError: + config[section][option] = 0 + + config["smtp"]["ssl"] = (config["smtp"]["ssl"].lower() == "true") + config["smtp"]["tls"] = (config["smtp"]["tls"].lower() == "true") + config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true") + config["email"]["short"] = (config["email"]["short"].lower() == "true") + config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true") + + # Migration + if config["scrub"]["percentage"]: + config["scrub"]["plan"] = config["scrub"]["percentage"] + + if args.scrub is not None: + config["scrub"]["enabled"] = args.scrub + + if args.ignore_deletethreshold: + config["snapraid"]["deletethreshold"] = -1 + + +def setup_logger(): + log_format = logging.Formatter( + "%(asctime)s [%(levelname)-6.6s] %(message)s") + root_logger = logging.getLogger() + logging.OUTPUT = 15 + logging.addLevelName(logging.OUTPUT, "OUTPUT") + logging.OUTERR = 25 + logging.addLevelName(logging.OUTERR, "OUTERR") + root_logger.setLevel(logging.OUTPUT) + console_logger = logging.StreamHandler(sys.stdout) + console_logger.setFormatter(log_format) + root_logger.addHandler(console_logger) + + if config["logging"]["file"]: + max_log_size = max(config["logging"]["maxsize"], 0) * 1024 + file_logger = logging.handlers.RotatingFileHandler( + config["logging"]["file"], + maxBytes=max_log_size, + backupCount=9) + file_logger.setFormatter(log_format) + root_logger.addHandler(file_logger) + + if config["email"]["sendon"]: + global email_log + email_log = StringIO() + email_logger = logging.StreamHandler(email_log) + email_logger.setFormatter(log_format) + if config["email"]["short"]: + # Don't send programm stdout in email + email_logger.setLevel(logging.INFO) + root_logger.addHandler(email_logger) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--conf", + default="snapraid-runner.conf", + metavar="CONFIG", + help="Configuration file (default: %(default)s)") + parser.add_argument("--no-scrub", action='store_false', + dest='scrub', default=None, + help="Do not scrub (overrides config)") + parser.add_argument("--ignore-deletethreshold", action='store_true', + help="Sync even if configured delete threshold is exceeded") + args = parser.parse_args() + + if not os.path.exists(args.conf): + print("snapraid-runner configuration file not found") + parser.print_help() + sys.exit(2) + + try: + load_config(args) + except Exception: + print("unexpected exception while loading config") + print(traceback.format_exc()) + sys.exit(2) + + try: + setup_logger() + except Exception: + print("unexpected exception while setting up logging") + print(traceback.format_exc()) + sys.exit(2) + + try: + run() + except Exception: + logging.exception("Run failed due to unexpected exception:") + finish(False) + + +def run(): + logging.info("=" * 60) + logging.info("Run started") + logging.info("=" * 60) + + if not os.path.isfile(config["snapraid"]["executable"]): + logging.error("The configured snapraid executable \"{}\" does not " + "exist or is not a file".format( + config["snapraid"]["executable"])) + finish(False) + if not os.path.isfile(config["snapraid"]["config"]): + logging.error("Snapraid config does not exist at " + + config["snapraid"]["config"]) + finish(False) + + if config["snapraid"]["touch"]: + logging.info("Running touch...") + snapraid_command("touch") + logging.info("*" * 60) + + logging.info("Running diff...") + diff_out = snapraid_command("diff", allow_statuscodes=[2]) + logging.info("*" * 60) + + diff_results = Counter(line.split(" ")[0] for line in diff_out) + diff_results = dict((x, diff_results[x]) for x in + ["add", "remove", "move", "update"]) + logging.info(("Diff results: {add} added, {remove} removed, " + + "{move} moved, {update} modified").format(**diff_results)) + + if (config["snapraid"]["deletethreshold"] >= 0 and + diff_results["remove"] > config["snapraid"]["deletethreshold"]): + logging.error( + "Deleted files exceed delete threshold of {}, aborting".format( + config["snapraid"]["deletethreshold"])) + logging.error("Run again with --ignore-deletethreshold to sync anyways") + finish(False) + + if (diff_results["remove"] + diff_results["add"] + diff_results["move"] + + diff_results["update"] == 0): + logging.info("No changes detected, no sync required") + else: + logging.info("Running sync...") + try: + snapraid_command("sync") + except subprocess.CalledProcessError as e: + logging.error(e) + finish(False) + logging.info("*" * 60) + + if config["scrub"]["enabled"]: + logging.info("Running scrub...") + try: + # Check if a percentage plan was given + int(config["scrub"]["plan"]) + except ValueError: + scrub_args = {"plan": config["scrub"]["plan"]} + else: + scrub_args = { + "plan": config["scrub"]["plan"], + "older-than": config["scrub"]["older-than"], + } + try: + snapraid_command("scrub", scrub_args) + except subprocess.CalledProcessError as e: + logging.error(e) + finish(False) + logging.info("*" * 60) + + logging.info("All done") + finish(True) + + +main() \ No newline at end of file