From 33c5ac3a72ba52512749ce2a4579b5d6efa00a1b Mon Sep 17 00:00:00 2001 From: 514fpv Date: Wed, 21 Feb 2024 23:25:06 +0800 Subject: [PATCH] package(tubesync): add tubesync application and module --- package/tubesync/database-local-socket.patch | 27 +++ package/tubesync/default.nix | 122 +++++++++++++ package/tubesync/gunicorn-env.patch | 19 ++ package/tubesync/nixos.nix | 172 +++++++++++++++++++ package/tubesync/state-dir-env.patch | 16 ++ 5 files changed, 356 insertions(+) create mode 100644 package/tubesync/database-local-socket.patch create mode 100644 package/tubesync/default.nix create mode 100644 package/tubesync/gunicorn-env.patch create mode 100644 package/tubesync/nixos.nix create mode 100644 package/tubesync/state-dir-env.patch diff --git a/package/tubesync/database-local-socket.patch b/package/tubesync/database-local-socket.patch new file mode 100644 index 00000000..b422f3ed --- /dev/null +++ b/package/tubesync/database-local-socket.patch @@ -0,0 +1,27 @@ +diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container +index a7a07ab..7564138 100644 +--- a/tubesync/tubesync/local_settings.py.container ++++ b/tubesync/tubesync/local_settings.py.container +@@ -34,14 +34,20 @@ if database_connection_env: + database_dict = parse_database_connection_string(database_connection_env) + + ++database_host = database_dict.get("HOST") ++if database_host == "localhost": ++ database_dict["HOST"] = None ++ database_dict["PASSWORD"] = None ++ ++ + if database_dict: + log.info(f'Using database connection: {database_dict["ENGINE"]}://' +- f'{database_dict["USER"]}:[hidden]@{database_dict["HOST"]}:' ++ f'{database_dict["USER"]}:[hidden]@{database_host}:' + f'{database_dict["PORT"]}/{database_dict["NAME"]}') + DATABASES = { + 'default': database_dict, + } +- DATABASE_CONNECTION_STR = (f'{database_dict["DRIVER"]} at "{database_dict["HOST"]}:' ++ DATABASE_CONNECTION_STR = (f'{database_dict["DRIVER"]} at "{database_host}:' + f'{database_dict["PORT"]}" database ' + f'"{database_dict["NAME"]}"') + else: diff --git a/package/tubesync/default.nix b/package/tubesync/default.nix new file mode 100644 index 00000000..c038690d --- /dev/null +++ b/package/tubesync/default.nix @@ -0,0 +1,122 @@ +{ lib +, stdenvNoCC +, ffmpeg +, callPackage +, fetchFromGitHub +, fetchPypi +, makeWrapper +, python3Packages }: with python3Packages; let + mkPypi = pname: version: src: format: buildPythonPackage { + inherit pname version src format; + doCheck = false; + nativeBuildInputs = [ setuptools ]; + }; + + mkPypi' = pname: version: hash: format: mkPypi pname version + (fetchPypi { + inherit pname version hash; + }) format; + + mkPypi'' = pname: version: hash: mkPypi' pname version hash + "setuptools"; + + django-compat = mkPypi'' "django-compat" "1.0.15" "sha256-OsmjvtxWuTZdnrJBvFFX0MGTdpv5lfmnjcG8JOfCMxs="; + django-appconf = mkPypi'' "django-appconf" "1.0.6" "sha256-z+h+qCfE7gS5pw+rkLhtcEywLymB+J2oQjyw+r+I778="; + django-basicauth = mkPypi'' "django-basicauth" "0.5.3" "sha256-FenjZvaY9TxxseeU2v6gYPmQoqxVa65rczDdJTJKCRw="; + django-sass-processor = mkPypi'' "django-sass-processor" "1.4" "sha256-sX850H06dRCuxCXBkZN+IwUC3ut8pr9pUKGt+LS3wcM="; + django-background-tasks = mkPypi'' "django-background-tasks" "1.2.5" "sha256-4bGejUlaJ2ydZMWh/4tBEy910vWORb5xt4ZQ2tWa+d4="; + + django-compressor = let + pname = "django-compressor"; + version = "4.4"; + in mkPypi pname version (fetchFromGitHub { + owner = pname; + repo = pname; + rev = "refs/tags/${version}"; + hash = "sha256-c9uS5Z077b23Aj8jV30XNsshbEfrLRX3ozXasitQ6UQ="; + }) "setuptools"; + + app = buildPythonApplication rec { + pname = "tubesync"; + version = "0.13.3"; + format = "other"; + + src = fetchFromGitHub { + name = "${pname}-src"; + owner = "meeb"; + repo = pname; + rev = "v${version}"; + hash = "sha256-33DDbECEn/3vrQ0qvxoz5OZ/y8bR6BZ2cYUtPsA7YYc="; + }; + + patches = [ + ./gunicorn-env.patch + ./state-dir-env.patch + ./database-local-socket.patch + ]; + + propagatedBuildInputs = [ + yt-dlp requests + httptools pillow + gunicorn whitenoise + psycopg2 mysqlclient + redis hiredis + libsass six + ] ++ [ + django_3 + django-compat + django-appconf + django-compressor + django-basicauth + django-sass-processor + django-background-tasks + ]; + + buildPhase = '' + mv "tubesync/tubesync/local_settings.py.container" "tubesync/tubesync/local_settings.py" + rm "tubesync/tubesync/local_settings.py.example" + rm "tubesync/tubesync/local_settings.py.container.orig" + + python3 tubesync/manage.py compilescss + python3 tubesync/manage.py collectstatic --no-input + ''; + + installPhase = '' + mkdir -p "$out" + cp -r "tubesync" "$out/app" + + FFMPEG_VERSION=$(${ffmpeg}/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }') + echo "ffmpeg_version = '$FFMPEG_VERSION'" >> "$out/app/common/third_party_versions.py" + + mv "$out/app/static" "$out/static" + ln -s "/tmp/tubesync/static" "$out/app/static" + ''; + }; +in stdenvNoCC.mkDerivation { + pname = "${app.pname}-wrapped"; + inherit (app) version; + + nativeBuildInputs = [ makeWrapper ]; + unpackPhase = "true"; + installPhase = '' + mkdir -p "$out/bin" + + makeWrapper "${python}/bin/python3" "$out/bin/tubesync-worker" \ + --chdir ${app}/app --add-flags \ + "${app}/app/manage.py process_tasks" \ + --set PATH ${lib.makeBinPath [ ffmpeg ]} + + makeWrapper "${gunicorn}/bin/gunicorn" "$out/bin/tubesync-gunicorn" \ + --chdir ${app}/app --add-flags \ + "-c ${app}/app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application" + + makeWrapper "${python}/bin/python3" "$out/bin/tubesync-migrate" \ + --chdir "${app}/app" --add-flags \ + "${app}/app/manage.py migrate" + ''; + + passthru = { + inherit app; + pythonPath = makePythonPath app.propagatedBuildInputs; + }; +} diff --git a/package/tubesync/gunicorn-env.patch b/package/tubesync/gunicorn-env.patch new file mode 100644 index 00000000..d1ee0965 --- /dev/null +++ b/package/tubesync/gunicorn-env.patch @@ -0,0 +1,19 @@ +diff --git a/tubesync/tubesync/gunicorn.py b/tubesync/tubesync/gunicorn.py +index d59c138..341af25 100644 +--- a/tubesync/tubesync/gunicorn.py ++++ b/tubesync/tubesync/gunicorn.py +@@ -23,11 +23,10 @@ def get_bind(): + + workers = get_num_workers() + timeout = 30 +-chdir = '/app' + daemon = False +-pidfile = '/run/app/gunicorn.pid' +-user = 'app' +-group = 'app' ++pidfile = os.getenv('GUNICORN_PID_FILE', '/var/run/tubesync/gunicorn.pid') ++user = os.getenv('GUNICORN_USER', 'tubesync') ++group = os.getenv('GUNICORN_GROUP', 'tubesync') + loglevel = 'info' + errorlog = '-' + accesslog = '/dev/null' # Access logs are printed to stdout from nginx diff --git a/package/tubesync/nixos.nix b/package/tubesync/nixos.nix new file mode 100644 index 00000000..efa6cbf5 --- /dev/null +++ b/package/tubesync/nixos.nix @@ -0,0 +1,172 @@ +{ pkgs +, lib +, config +, ... }: with lib; let + cfg = config.services.tubesync; +in { + options.services.tubesync = { + enable = mkEnableOption "tubesync stack"; + debug = mkEnableOption "debug logging"; + package = mkOption { + type = with types; package; + default = pkgs.tubesync; + description = "tubesync launcher package"; + }; + + user = mkOption { + type = with types; str; + default = "tubesync"; + description = "user under which tubesync runs"; + }; + group = mkOption { + type = with types; str; + default = "tubesync"; + description = "group under which tubesync runs"; + }; + + listen = { + host = mkOption { + type = with types; str; + default = "127.0.0.1"; + description = "host to listen on"; + }; + port = mkOption { + type = with types; port; + default = 8080; + description = "port to listen on"; + }; + }; + + stateDir = mkOption { + type = with types; str; + default = "/var/lib/tubesync"; + description = "path to tubesync state storage directory"; + }; + + dataDir = mkOption { + type = with types; str; + default = "${cfg.stateDir}/downloads"; + description = "path to tubesync video downloads"; + }; + + database = mkOption { + type = with types; str; + default = "postgresql://tubesync:@localhost:5432/tubesync"; + description = "database connection string"; + }; + }; + + config = mkIf cfg.enable { + systemd.services = let + env = { + PYTHONPATH = cfg.package.pythonPath; + GUNICORN_PID_FILE = "${cfg.stateDir}/run/gunicorn.pid"; + GUNICORN_USER = cfg.user; + GUNICORN_GROUP = cfg.group; + DATABASE_CONNECTION = cfg.database; + CONFIG_BASE_DIR = cfg.stateDir; + DOWNLOADS_BASE_DIR = cfg.dataDir; + TUBESYNC_DEBUG = mkIf cfg.debug "True"; + + REDIS_CONNECTION = "redis+socket://" + + "${cfg.stateDir}/run/redis.sock"; + }; + + base = description: { + description = "tubesync: ${description}"; + wantedBy = [ "multi-user.target" ]; + environment = env; + path = [ cfg.package ]; + serviceConfig = { + WorkingDirectory = cfg.stateDir; + User = cfg.user; + Group = cfg.group; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateTmp = true; + PrivateDevices = true; + PrivateUsers = false; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProcSubset = "all"; + ProtectSystem = "strict"; + RemoveIPC = true; + ReadWritePaths = with cfg; [ stateDir dataDir ]; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_NETLINK" + "AF_UNIX" + ]; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "@chown" + ]; + UMask = "0077"; + }; + }; + + base' = description: (base description) // { + after = [ "tubesync.service" ]; + partOf = [ "tubesync.service" ]; + }; + in { + tubesync = recursiveUpdate (base "gunicorn") ({ + after = [ "network.target" ]; + serviceConfig = { + PIDFile = env.GUNICORN_PID_FILE; + ExecStartPre = pkgs.writeShellScript "tubesync-setup" '' + set -xe + tubesync-migrate + + mkdir -p "/tmp/tubesync" + cp -r "${cfg.package.app}/static/." "/tmp/tubesync/static" + chmod +w -R "/tmp/tubesync/static" + ''; + ExecStart = "${cfg.package}/bin/tubesync-gunicorn"; + ExecReload = "/usr/bin/env kill -s HUP $MAINPID"; + ExecStop = "/usr/bin/env kill -s TERM $MAINPID"; + ExecStopPost = pkgs.writeShellScript "tubesync-cleanup" '' + rm -f "$GUNICORN_PID_FILE" + rm -rf "/tmp/tubesync" + ''; + }; + }); + + tubesync-worker = recursiveUpdate (base' "worker") ({ + serviceConfig.ExecStart = "${cfg.package}/bin/tubesync-worker"; + }); + }; + + services.redis.servers.tubesync-celery = { + enable = true; + inherit (cfg) user; + unixSocket = "${cfg.stateDir}/run/redis.sock"; + save = [ ]; + }; + + users.users = mkIf (cfg.user == "tubesync") { + tubesync = { + description = "tubesync service account"; + group = cfg.group; + uid = config.ids.uids.tubesync; + }; + }; + + users.groups = mkIf (cfg.group == "tubesync") { + tubesync.gid = config.ids.gids.tubesync; + }; + + ids.uids.tubesync = 101; + ids.gids.tubesync = 101; + }; +} diff --git a/package/tubesync/state-dir-env.patch b/package/tubesync/state-dir-env.patch new file mode 100644 index 00000000..f7bb1d2d --- /dev/null +++ b/package/tubesync/state-dir-env.patch @@ -0,0 +1,16 @@ +diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container +index a7a07ab..9207c7f 100644 +--- a/tubesync/tubesync/local_settings.py.container ++++ b/tubesync/tubesync/local_settings.py.container +@@ -6,9 +6,8 @@ from common.utils import parse_database_connection_string + + + BASE_DIR = Path(__file__).resolve().parent.parent +-ROOT_DIR = Path('/') +-CONFIG_BASE_DIR = ROOT_DIR / 'config' +-DOWNLOADS_BASE_DIR = ROOT_DIR / 'downloads' ++CONFIG_BASE_DIR = Path(os.getenv('CONFIG_BASE_DIR', "/var/lib/tubesync")) ++DOWNLOADS_BASE_DIR = Path(os.getenv('DOWNLOADS_BASE_DIR', f"{CONFIG_BASE_DIR}/downloads")) + DJANGO_URL_PREFIX = os.getenv('DJANGO_URL_PREFIX', None) + STATIC_URL = str(os.getenv('DJANGO_STATIC_URL', '/static/')) + if DJANGO_URL_PREFIX and STATIC_URL: