feat(tubesync): add tubesync application and module

package/tubesync/database-local-socket.patch

@@ -0,0 +1,27 @@
+diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container
+index a7a07ab..7564138 100644
+--- a/tubesync/tubesync/local_settings.py.container
++++ b/tubesync/tubesync/local_settings.py.container
+@@ -34,14 +34,20 @@ if database_connection_env:
+     database_dict = parse_database_connection_string(database_connection_env)
+ 
+ 
++database_host = database_dict.get("HOST")
++if database_host == "localhost":
++    database_dict["HOST"] = None
++    database_dict["PASSWORD"] = None
++
++
+ if database_dict:
+     log.info(f'Using database connection: {database_dict["ENGINE"]}://'
+-             f'{database_dict["USER"]}:[hidden]@{database_dict["HOST"]}:'
++             f'{database_dict["USER"]}:[hidden]@{database_host}:'
+              f'{database_dict["PORT"]}/{database_dict["NAME"]}')
+     DATABASES = {
+         'default': database_dict,
+     }
+-    DATABASE_CONNECTION_STR = (f'{database_dict["DRIVER"]} at "{database_dict["HOST"]}:'
++    DATABASE_CONNECTION_STR = (f'{database_dict["DRIVER"]} at "{database_host}:'
+                                f'{database_dict["PORT"]}" database '
+                                f'"{database_dict["NAME"]}"')
+ else:

package/tubesync/default.nix

@@ -0,0 +1,122 @@
+{ lib
+, stdenvNoCC
+, ffmpeg
+, callPackage
+, fetchFromGitHub
+, fetchPypi
+, makeWrapper
+, python3Packages }: with python3Packages; let
+  mkPypi = pname: version: src: format: buildPythonPackage {
+    inherit pname version src format;
+    doCheck = false;
+    nativeBuildInputs = [ setuptools ];
+  };
+
+  mkPypi' = pname: version: hash: format: mkPypi pname version
+  (fetchPypi {
+    inherit pname version hash;
+  }) format;
+
+  mkPypi'' = pname: version: hash: mkPypi' pname version hash
+  "setuptools";
+
+  django-compat = mkPypi'' "django-compat" "1.0.15" "sha256-OsmjvtxWuTZdnrJBvFFX0MGTdpv5lfmnjcG8JOfCMxs=";
+  django-appconf = mkPypi'' "django-appconf" "1.0.6" "sha256-z+h+qCfE7gS5pw+rkLhtcEywLymB+J2oQjyw+r+I778=";
+  django-basicauth = mkPypi'' "django-basicauth" "0.5.3" "sha256-FenjZvaY9TxxseeU2v6gYPmQoqxVa65rczDdJTJKCRw=";
+  django-sass-processor = mkPypi'' "django-sass-processor" "1.4" "sha256-sX850H06dRCuxCXBkZN+IwUC3ut8pr9pUKGt+LS3wcM=";
+  django-background-tasks = mkPypi'' "django-background-tasks" "1.2.5" "sha256-4bGejUlaJ2ydZMWh/4tBEy910vWORb5xt4ZQ2tWa+d4=";
+
+  django-compressor = let
+    pname = "django-compressor";
+    version = "4.4";
+  in mkPypi pname version (fetchFromGitHub {
+    owner = pname;
+    repo = pname;
+    rev = "refs/tags/${version}";
+    hash = "sha256-c9uS5Z077b23Aj8jV30XNsshbEfrLRX3ozXasitQ6UQ=";
+  }) "setuptools";
+
+  app = buildPythonApplication rec {
+    pname = "tubesync";
+    version = "0.13.3";
+    format = "other";
+
+    src = fetchFromGitHub {
+      name = "${pname}-src";
+      owner = "meeb";
+      repo = pname;
+      rev = "v${version}";
+      hash = "sha256-33DDbECEn/3vrQ0qvxoz5OZ/y8bR6BZ2cYUtPsA7YYc=";
+    };
+
+    patches = [
+      ./gunicorn-env.patch
+      ./state-dir-env.patch
+      ./database-local-socket.patch
+    ];
+
+    propagatedBuildInputs = [
+      yt-dlp requests
+      httptools pillow
+      gunicorn whitenoise
+      psycopg2 mysqlclient
+      redis hiredis
+      libsass six
+    ] ++ [
+      django_3
+      django-compat
+      django-appconf
+      django-compressor
+      django-basicauth
+      django-sass-processor
+      django-background-tasks
+    ];
+
+    buildPhase = ''
+      mv "tubesync/tubesync/local_settings.py.container" "tubesync/tubesync/local_settings.py"
+      rm "tubesync/tubesync/local_settings.py.example"
+      rm "tubesync/tubesync/local_settings.py.container.orig"
+
+      python3 tubesync/manage.py compilescss
+      python3 tubesync/manage.py collectstatic --no-input
+    '';
+
+    installPhase = ''
+      mkdir -p "$out"
+      cp -r "tubesync" "$out/app"
+
+      FFMPEG_VERSION=$(${ffmpeg}/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }')
+      echo "ffmpeg_version = '$FFMPEG_VERSION'" >> "$out/app/common/third_party_versions.py"
+
+      mv "$out/app/static" "$out/static"
+      ln -s "/tmp/tubesync/static" "$out/app/static"
+    '';
+  };
+in stdenvNoCC.mkDerivation {
+  pname = "${app.pname}-wrapped";
+  inherit (app) version;
+
+  nativeBuildInputs = [ makeWrapper ];
+  unpackPhase = "true";
+  installPhase = ''
+    mkdir -p "$out/bin"
+
+    makeWrapper "${python}/bin/python3" "$out/bin/tubesync-worker" \
+    --chdir ${app}/app --add-flags \
+    "${app}/app/manage.py process_tasks" \
+    --set PATH ${lib.makeBinPath [ ffmpeg ]}
+
+    makeWrapper "${gunicorn}/bin/gunicorn" "$out/bin/tubesync-gunicorn" \
+    --chdir ${app}/app --add-flags \
+    "-c ${app}/app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application"
+
+    makeWrapper "${python}/bin/python3" "$out/bin/tubesync-migrate" \
+    --chdir "${app}/app" --add-flags \
+    "${app}/app/manage.py migrate"
+  '';
+
+  passthru = {
+    inherit app;
+    pythonPath = makePythonPath app.propagatedBuildInputs;
+  };
+}

package/tubesync/gunicorn-env.patch

@@ -0,0 +1,19 @@
+diff --git a/tubesync/tubesync/gunicorn.py b/tubesync/tubesync/gunicorn.py
+index d59c138..341af25 100644
+--- a/tubesync/tubesync/gunicorn.py
++++ b/tubesync/tubesync/gunicorn.py
+@@ -23,11 +23,10 @@ def get_bind():
+ 
+ workers = get_num_workers()
+ timeout = 30
+-chdir = '/app'
+ daemon = False
+-pidfile = '/run/app/gunicorn.pid'
+-user = 'app'
+-group = 'app'
++pidfile = os.getenv('GUNICORN_PID_FILE', '/var/run/tubesync/gunicorn.pid')
++user = os.getenv('GUNICORN_USER', 'tubesync')
++group = os.getenv('GUNICORN_GROUP', 'tubesync')
+ loglevel = 'info'
+ errorlog = '-'
+ accesslog = '/dev/null'  # Access logs are printed to stdout from nginx

package/tubesync/nixos.nix

@@ -0,0 +1,172 @@
+{ pkgs
+, lib
+, config
+, ... }: with lib; let
+  cfg = config.services.tubesync;
+in {
+  options.services.tubesync = {
+    enable = mkEnableOption "tubesync stack";
+    debug = mkEnableOption "debug logging";
+    package = mkOption {
+      type = with types; package;
+      default = pkgs.tubesync;
+      description = "tubesync launcher package";
+    };
+
+    user = mkOption {
+      type = with types; str;
+      default = "tubesync";
+      description = "user under which tubesync runs";
+    };
+    group = mkOption {
+      type = with types; str;
+      default = "tubesync";
+      description = "group under which tubesync runs";
+    };
+
+    listen = {
+      host = mkOption {
+        type = with types; str;
+        default = "127.0.0.1";
+        description = "host to listen on";
+      };
+      port = mkOption {
+        type = with types; port;
+        default = 8080;
+        description = "port to listen on";
+      };
+    };
+
+    stateDir = mkOption {
+      type = with types; str;
+      default = "/var/lib/tubesync";
+      description = "path to tubesync state storage directory";
+    };
+
+    dataDir = mkOption {
+      type = with types; str;
+      default = "${cfg.stateDir}/downloads";
+      description = "path to tubesync video downloads";
+    };
+
+    database = mkOption {
+      type = with types; str;
+      default = "postgresql://tubesync:@localhost:5432/tubesync";
+      description = "database connection string";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services = let
+      env = {
+        PYTHONPATH = cfg.package.pythonPath;
+        GUNICORN_PID_FILE = "${cfg.stateDir}/run/gunicorn.pid";
+        GUNICORN_USER = cfg.user;
+        GUNICORN_GROUP = cfg.group;
+        DATABASE_CONNECTION = cfg.database;
+        CONFIG_BASE_DIR = cfg.stateDir;
+        DOWNLOADS_BASE_DIR = cfg.dataDir;
+        TUBESYNC_DEBUG = mkIf cfg.debug "True";
+
+        REDIS_CONNECTION = "redis+socket://"
+        + "${cfg.stateDir}/run/redis.sock";
+      };
+
+      base = description: {
+        description = "tubesync: ${description}";
+        wantedBy = [ "multi-user.target" ];
+        environment = env;
+        path = [ cfg.package ];
+        serviceConfig = {
+          WorkingDirectory = cfg.stateDir;
+          User = cfg.user;
+          Group = cfg.group;
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          NoNewPrivileges = true;
+          PrivateTmp = true;
+          PrivateDevices = true;
+          PrivateUsers = false;
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectProc = "invisible";
+          ProcSubset = "all";
+          ProtectSystem = "strict";
+          RemoveIPC = true;
+          ReadWritePaths = with cfg; [ stateDir dataDir ];
+          RestrictAddressFamilies = [
+            "AF_INET"
+            "AF_INET6"
+            "AF_NETLINK"
+            "AF_UNIX"
+          ];
+          SystemCallArchitectures = "native";
+          SystemCallFilter = [
+            "@system-service"
+            "~@privileged"
+            "@chown"
+          ];
+          UMask = "0077";
+        };
+      };
+
+      base' = description: (base description) // {
+        after = [ "tubesync.service" ];
+        partOf = [ "tubesync.service" ];
+      };
+    in {
+      tubesync = recursiveUpdate (base "gunicorn") ({
+        after = [ "network.target" ];
+        serviceConfig = {
+          PIDFile = env.GUNICORN_PID_FILE;
+          ExecStartPre = pkgs.writeShellScript "tubesync-setup" ''
+            set -xe
+            tubesync-migrate
+
+            mkdir -p "/tmp/tubesync"
+            cp -r "${cfg.package.app}/static/." "/tmp/tubesync/static"
+            chmod +w -R "/tmp/tubesync/static"
+          '';
+          ExecStart = "${cfg.package}/bin/tubesync-gunicorn";
+          ExecReload = "/usr/bin/env kill -s HUP $MAINPID";
+          ExecStop = "/usr/bin/env kill -s TERM $MAINPID";
+          ExecStopPost = pkgs.writeShellScript "tubesync-cleanup" ''
+            rm -f "$GUNICORN_PID_FILE"
+            rm -rf "/tmp/tubesync"
+          '';
+        };
+      });
+
+      tubesync-worker = recursiveUpdate (base' "worker") ({
+        serviceConfig.ExecStart = "${cfg.package}/bin/tubesync-worker";
+      });
+    };
+
+    services.redis.servers.tubesync-celery = {
+      enable = true;
+      inherit (cfg) user;
+      unixSocket = "${cfg.stateDir}/run/redis.sock";
+      save = [ ];
+    };
+
+    users.users = mkIf (cfg.user == "tubesync") {
+      tubesync = {
+        description = "tubesync service account";
+        group = cfg.group;
+        uid = config.ids.uids.tubesync;
+      };
+    };
+
+    users.groups = mkIf (cfg.group == "tubesync") {
+      tubesync.gid = config.ids.gids.tubesync;
+    };
+
+    ids.uids.tubesync = 101;
+    ids.gids.tubesync = 101;
+  };
+}

package/tubesync/state-dir-env.patch

@@ -0,0 +1,16 @@
+diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container
+index a7a07ab..9207c7f 100644
+--- a/tubesync/tubesync/local_settings.py.container
++++ b/tubesync/tubesync/local_settings.py.container
+@@ -6,9 +6,8 @@ from common.utils import parse_database_connection_string
+ 
+ 
+ BASE_DIR = Path(__file__).resolve().parent.parent
+-ROOT_DIR = Path('/')
+-CONFIG_BASE_DIR = ROOT_DIR / 'config'
+-DOWNLOADS_BASE_DIR = ROOT_DIR / 'downloads'
++CONFIG_BASE_DIR = Path(os.getenv('CONFIG_BASE_DIR', "/var/lib/tubesync"))
++DOWNLOADS_BASE_DIR = Path(os.getenv('DOWNLOADS_BASE_DIR', f"{CONFIG_BASE_DIR}/downloads"))
+ DJANGO_URL_PREFIX = os.getenv('DJANGO_URL_PREFIX', None)
+ STATIC_URL = str(os.getenv('DJANGO_STATIC_URL', '/static/'))
+ if DJANGO_URL_PREFIX and STATIC_URL: