Custom pkg for snapraid-runner

needs more configuration once snapraid itself is setup

flake.nix

@@ -13,7 +13,10 @@
     let
       system = "x86_64-linux";
       lib = nixpkgs.lib;
-      pkgs = nixpkgs.legacyPackages.${system};
+      pkgs = import nixpkgs {
+        inherit system;
+        overlays = [ (import ./overlays) ];
+      };
     in {
       nixosConfigurations = {
         cloud = lib.nixosSystem {

nixos/configuration.nix

@@ -12,11 +12,16 @@ in {
       "${modulesPath}/virtualisation/lxc-container.nix"
       # Include the container-specific autogenerated configuration.
       #./lxd.nix - this has to be commented out from the system tarball
+
+      # Snapraid-runner
+      ./imports/snapraid-runner.nix
       
       # Import hardware configuration.
       ./hardware-configuration.nix
     ];
-  
+
+  nixpkgs.overlays = [ (import ./overlays) ];
+
   # NETWORKING
   networking = {
     firewall = {
@@ -91,7 +96,9 @@ in {
     openssh
     ranger
     sshfs
-    wget
+    snapraid
+    snapraid-runner 
+    wget 
   ];
 
   # PROGRAMS & SERVICES

nixos/imports/snapraid-runner.nix

@@ -0,0 +1,5 @@
+{ pkgs, ... }:
+
+{
+    environment.etc."snapraid-runner.conf".text = builtins.readFile ../pkgs/snapraid-runner/snapraid-runner.conf;
+}

nixos/overlays/default.nix

@@ -0,0 +1,7 @@
+self: super:
+
+let
+    callPackage = super.callPackage;
+in {
+    snapraid-runner = callPackage ../pkgs/snapraid-runner { };
+}

nixos/pkgs/snapraid-runner/default.nix

@@ -0,0 +1,56 @@
+{
+  stdenv,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  snapraid-runner-py = pkgs.writeTextFile {
+    name = "snapraid-runner.py";
+    executable = true;
+    destination = "/bin/snapraid-runner.py";
+    text = builtins.readFile ./snapraid-runner.py;
+  };
+
+  snapraid-runner = pkgs.writeTextFile {
+    name = "snapraid-runner.sh";
+    executable = true;
+    destination = "/bin/snapraid-runner";
+    text = ''#!${pkgs.stdenv.shell}
+
+# Check if the "-c" option is present
+config_option_present=false
+for arg in "$@"; do
+  if [ "$arg" = "-c" ]; then
+    config_option_present=true
+    break
+  fi
+done
+
+# Add the default config option if not present
+if [ "$config_option_present" = false ]; then
+  set -- "-c" "/etc/snapraid-runner.conf" "$@"
+fi
+
+${pkgs.python311}/bin/python3 ${snapraid-runner-py}/bin/snapraid-runner.py "$@"
+'';
+  };
+
+in
+stdenv.mkDerivation rec {
+  pname = "snapraid-runner";
+  version = "8f78f9f1af8ca5a9b6469a6c142cab2577157331";
+
+  buildInputs = [
+    snapraid-runner
+  ];
+
+  builder = pkgs.writeTextFile {
+    name = "builder.sh";
+    text = ''. $stdenv/setup
+mkdir -p $out/bin
+ln -sf ${snapraid-runner}/bin/snapraid-runner $out/bin/snapraid-runner'';
+  }; 
+}
+

nixos/pkgs/snapraid-runner/snapraid-runner.conf

@@ -0,0 +1,44 @@
+[snapraid]
+; path to the snapraid executable (e.g. /bin/snapraid)
+executable = /usr/bin/snapraid
+; path to the snapraid config to be used
+config = /etc/snapraid.conf
+; abort operation if there are more deletes than this, set to -1 to disable
+deletethreshold = 40
+; if you want touch to be ran each time
+touch = false
+
+[logging]
+; logfile to write to, leave empty to disable
+file = /var/log/snapraid-runner.log
+; maximum logfile size in KiB, leave empty for infinite
+maxsize = 5000
+
+[email]
+; when to send an email, comma-separated list of [success, error]
+sendon = success,error
+; set to false to get full programm output via email
+short = true
+subject = [SnapRAID] Status Report:
+from = cloud@ryot.foo
+to = [REDACTED]
+; maximum email size in KiB
+maxsize = 500
+
+[smtp]
+host = ryot.foo
+; leave empty for default port
+port =
+; set to "true" to activate
+ssl = true
+tls = true
+user = admin
+password = [REDACTED]
+
+[scrub]
+; set to true to run scrub after sync
+enabled = true
+; scrub plan - either a percentage or one of [bad, new, full]
+plan = 12
+; minimum block age (in days) for scrubbing. Only used with percentage plans
+older-than = 10

nixos/pkgs/snapraid-runner/snapraid-runner.py

@@ -0,0 +1,314 @@
+#!/usr/bin/env python3
+import argparse
+import configparser
+import logging
+import logging.handlers
+import os.path
+import subprocess
+import sys
+import threading
+import time
+import traceback
+from collections import Counter, defaultdict
+from io import StringIO
+
+# Global variables
+config = None
+email_log = None
+
+
+def tee_log(infile, out_lines, log_level):
+    """
+    Create a thread that saves all the output on infile to out_lines and
+    logs every line with log_level
+    """
+    def tee_thread():
+        for line in iter(infile.readline, ""):
+            logging.log(log_level, line.rstrip())
+            out_lines.append(line)
+        infile.close()
+    t = threading.Thread(target=tee_thread)
+    t.daemon = True
+    t.start()
+    return t
+
+
+def snapraid_command(command, args={}, *, allow_statuscodes=[]):
+    """
+    Run snapraid command
+    Raises subprocess.CalledProcessError if errorlevel != 0
+    """
+    arguments = ["--conf", config["snapraid"]["config"],
+                    "--quiet"]
+    for (k, v) in args.items():
+        arguments.extend(["--" + k, str(v)])
+    p = subprocess.Popen(
+        [config["snapraid"]["executable"], command] + arguments,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        # Snapraid always outputs utf-8 on windows. On linux, utf-8
+        # also seems a sensible assumption.
+        encoding="utf-8",
+        errors="replace")
+    out = []
+    threads = [
+        tee_log(p.stdout, out, logging.OUTPUT),
+        tee_log(p.stderr, [], logging.OUTERR)]
+    for t in threads:
+        t.join()
+    ret = p.wait()
+    # sleep for a while to make pervent output mixup
+    time.sleep(0.3)
+    if ret == 0 or ret in allow_statuscodes:
+        return out
+    else:
+        raise subprocess.CalledProcessError(ret, "snapraid " + command)
+
+
+def send_email(success):
+    import smtplib
+    from email.mime.text import MIMEText
+    from email import charset
+
+    if len(config["smtp"]["host"]) == 0:
+        logging.error("Failed to send email because smtp host is not set")
+        return
+
+    # use quoted-printable instead of the default base64
+    charset.add_charset("utf-8", charset.SHORTEST, charset.QP)
+    if success:
+        body = "SnapRAID job completed successfully:\n\n\n"
+    else:
+        body = "Error during SnapRAID job:\n\n\n"
+
+    log = email_log.getvalue()
+    maxsize = config['email'].get('maxsize', 500) * 1024
+    if maxsize and len(log) > maxsize:
+        cut_lines = log.count("\n", maxsize // 2, -maxsize // 2)
+        log = (
+            "NOTE: Log was too big for email and was shortened\n\n" +
+            log[:maxsize // 2] +
+            "[...]\n\n\n --- LOG WAS TOO BIG - {} LINES REMOVED --\n\n\n[...]".format(
+                cut_lines) +
+            log[-maxsize // 2:])
+    body += log
+
+    msg = MIMEText(body, "plain", "utf-8")
+    msg["Subject"] = config["email"]["subject"] + \
+        (" SUCCESS" if success else " ERROR")
+    msg["From"] = config["email"]["from"]
+    msg["To"] = config["email"]["to"]
+    smtp = {"host": config["smtp"]["host"]}
+    if config["smtp"]["port"]:
+        smtp["port"] = config["smtp"]["port"]
+    if config["smtp"]["ssl"]:
+        server = smtplib.SMTP_SSL(**smtp)
+    else:
+        server = smtplib.SMTP(**smtp)
+        if config["smtp"]["tls"]:
+            server.starttls()
+    if config["smtp"]["user"]:
+        server.login(config["smtp"]["user"], config["smtp"]["password"])
+    server.sendmail(
+        config["email"]["from"],
+        [config["email"]["to"]],
+        msg.as_string())
+    server.quit()
+
+
+def finish(is_success):
+    if ("error", "success")[is_success] in config["email"]["sendon"]:
+        try:
+            send_email(is_success)
+        except Exception:
+            logging.exception("Failed to send email")
+    if is_success:
+        logging.info("Run finished successfully")
+    else:
+        logging.error("Run failed")
+    sys.exit(0 if is_success else 1)
+
+
+def load_config(args):
+    global config
+    parser = configparser.RawConfigParser()
+    parser.read(args.conf)
+    sections = ["snapraid", "logging", "email", "smtp", "scrub"]
+    config = dict((x, defaultdict(lambda: "")) for x in sections)
+    for section in parser.sections():
+        for (k, v) in parser.items(section):
+            config[section][k] = v.strip()
+
+    int_options = [
+        ("snapraid", "deletethreshold"), ("logging", "maxsize"),
+        ("scrub", "older-than"), ("email", "maxsize"),
+    ]
+    for section, option in int_options:
+        try:
+            config[section][option] = int(config[section][option])
+        except ValueError:
+            config[section][option] = 0
+
+    config["smtp"]["ssl"] = (config["smtp"]["ssl"].lower() == "true")
+    config["smtp"]["tls"] = (config["smtp"]["tls"].lower() == "true")
+    config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true")
+    config["email"]["short"] = (config["email"]["short"].lower() == "true")
+    config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true")
+
+    # Migration
+    if config["scrub"]["percentage"]:
+        config["scrub"]["plan"] = config["scrub"]["percentage"]
+
+    if args.scrub is not None:
+        config["scrub"]["enabled"] = args.scrub
+
+    if args.ignore_deletethreshold:
+        config["snapraid"]["deletethreshold"] = -1
+
+
+def setup_logger():
+    log_format = logging.Formatter(
+        "%(asctime)s [%(levelname)-6.6s] %(message)s")
+    root_logger = logging.getLogger()
+    logging.OUTPUT = 15
+    logging.addLevelName(logging.OUTPUT, "OUTPUT")
+    logging.OUTERR = 25
+    logging.addLevelName(logging.OUTERR, "OUTERR")
+    root_logger.setLevel(logging.OUTPUT)
+    console_logger = logging.StreamHandler(sys.stdout)
+    console_logger.setFormatter(log_format)
+    root_logger.addHandler(console_logger)
+
+    if config["logging"]["file"]:
+        max_log_size = max(config["logging"]["maxsize"], 0) * 1024
+        file_logger = logging.handlers.RotatingFileHandler(
+            config["logging"]["file"],
+            maxBytes=max_log_size,
+            backupCount=9)
+        file_logger.setFormatter(log_format)
+        root_logger.addHandler(file_logger)
+
+    if config["email"]["sendon"]:
+        global email_log
+        email_log = StringIO()
+        email_logger = logging.StreamHandler(email_log)
+        email_logger.setFormatter(log_format)
+        if config["email"]["short"]:
+            # Don't send programm stdout in email
+            email_logger.setLevel(logging.INFO)
+        root_logger.addHandler(email_logger)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-c", "--conf",
+                        default="snapraid-runner.conf",
+                        metavar="CONFIG",
+                        help="Configuration file (default: %(default)s)")
+    parser.add_argument("--no-scrub", action='store_false',
+                        dest='scrub', default=None,
+                        help="Do not scrub (overrides config)")
+    parser.add_argument("--ignore-deletethreshold", action='store_true',
+                        help="Sync even if configured delete threshold is exceeded")
+    args = parser.parse_args()
+
+    if not os.path.exists(args.conf):
+        print("snapraid-runner configuration file not found")
+        parser.print_help()
+        sys.exit(2)
+
+    try:
+        load_config(args)
+    except Exception:
+        print("unexpected exception while loading config")
+        print(traceback.format_exc())
+        sys.exit(2)
+
+    try:
+        setup_logger()
+    except Exception:
+        print("unexpected exception while setting up logging")
+        print(traceback.format_exc())
+        sys.exit(2)
+
+    try:
+        run()
+    except Exception:
+        logging.exception("Run failed due to unexpected exception:")
+        finish(False)
+
+
+def run():
+    logging.info("=" * 60)
+    logging.info("Run started")
+    logging.info("=" * 60)
+
+    if not os.path.isfile(config["snapraid"]["executable"]):
+        logging.error("The configured snapraid executable \"{}\" does not "
+                        "exist or is not a file".format(
+                            config["snapraid"]["executable"]))
+        finish(False)
+    if not os.path.isfile(config["snapraid"]["config"]):
+        logging.error("Snapraid config does not exist at " +
+                        config["snapraid"]["config"])
+        finish(False)
+
+    if config["snapraid"]["touch"]:
+        logging.info("Running touch...")
+        snapraid_command("touch")
+        logging.info("*" * 60)
+
+    logging.info("Running diff...")
+    diff_out = snapraid_command("diff", allow_statuscodes=[2])
+    logging.info("*" * 60)
+
+    diff_results = Counter(line.split(" ")[0] for line in diff_out)
+    diff_results = dict((x, diff_results[x]) for x in
+                        ["add", "remove", "move", "update"])
+    logging.info(("Diff results: {add} added,  {remove} removed,  " +
+                    "{move} moved,  {update} modified").format(**diff_results))
+
+    if (config["snapraid"]["deletethreshold"] >= 0 and
+            diff_results["remove"] > config["snapraid"]["deletethreshold"]):
+        logging.error(
+            "Deleted files exceed delete threshold of {}, aborting".format(
+                config["snapraid"]["deletethreshold"]))
+        logging.error("Run again with --ignore-deletethreshold to sync anyways")
+        finish(False)
+
+    if (diff_results["remove"] + diff_results["add"] + diff_results["move"] +
+            diff_results["update"] == 0):
+        logging.info("No changes detected, no sync required")
+    else:
+        logging.info("Running sync...")
+        try:
+            snapraid_command("sync")
+        except subprocess.CalledProcessError as e:
+            logging.error(e)
+            finish(False)
+        logging.info("*" * 60)
+
+    if config["scrub"]["enabled"]:
+        logging.info("Running scrub...")
+        try:
+            # Check if a percentage plan was given
+            int(config["scrub"]["plan"])
+        except ValueError:
+            scrub_args = {"plan": config["scrub"]["plan"]}
+        else:
+            scrub_args = {
+                "plan": config["scrub"]["plan"],
+                "older-than": config["scrub"]["older-than"],
+            }
+        try:
+            snapraid_command("scrub", scrub_args)
+        except subprocess.CalledProcessError as e:
+            logging.error(e)
+            finish(False)
+        logging.info("*" * 60)
+
+    logging.info("All done")
+    finish(True)
+
+
+main()