From 19936eaeb9dae6012a962de8d9a6e2703ef4727c Mon Sep 17 00:00:00 2001 From: Kerstin Humm Date: Mon, 12 Aug 2024 15:43:39 +0200 Subject: [PATCH] weblate: sync module as far as possible with NixOS PR https://github.com/NixOS/nixpkgs/pull/325541 --- pkgs/by-name/weblate/package.nix | 56 +++- projects/Weblate/examples/base.nix | 4 +- projects/Weblate/service.nix | 331 +++++++++----------- projects/Weblate/tests/integration-test.nix | 42 +-- 4 files changed, 211 insertions(+), 222 deletions(-) diff --git a/pkgs/by-name/weblate/package.nix b/pkgs/by-name/weblate/package.nix index c1df29f3..1b6afc12 100644 --- a/pkgs/by-name/weblate/package.nix +++ b/pkgs/by-name/weblate/package.nix @@ -2,7 +2,9 @@ stdenv, lib, fetchFromGitHub, + writeText, poetry2nix, + python3, pkg-config, openssl, isocodes, @@ -12,11 +14,15 @@ postgresql, leptonica, tesseract, - gobject-introspection, - wrapGAppsNoGuiHook, zlib, + pango, + harfbuzz, + librsvg, + gdk-pixbuf, + glib, + git, }: -poetry2nix.mkPoetryApplication { +poetry2nix.mkPoetryApplication rec { src = fetchFromGitHub { owner = "WeblateOrg"; repo = "weblate"; @@ -27,21 +33,25 @@ poetry2nix.mkPoetryApplication { pyproject = ./pyproject.toml; poetrylock = ./poetry.lock; + outputs = [ + "out" + "static" + ]; + patches = [ # FIXME This shouldn't be necessary and probably has to do with some dependency mismatch. ./cache.lock.patch ]; - makeWrapperArgs = [ - "\${gappsWrapperArgs[@]}" - ]; - - nativeBuildInputs = [ - wrapGAppsNoGuiHook - gobject-introspection + # We don't just use wrapGAppsNoGuiHook because we need to expose GI_TYPELIB_PATH + GI_TYPELIB_PATH = lib.makeSearchPathOutput "out" "lib/girepository-1.0" [ + pango + harfbuzz + librsvg + gdk-pixbuf + glib ]; - - dontWrapGApps = true; + makeWrapperArgs = ["--set GI_TYPELIB_PATH \"$GI_TYPELIB_PATH\""]; overrides = poetry2nix.overrides.withDefaults ( self: super: { @@ -153,6 +163,28 @@ poetry2nix.mkPoetryApplication { } ); + nativeBuildInputs = [git]; + + # Build static files into a separate output + postBuild = let + staticSettings = writeText "static_settings.py" '' + STATIC_ROOT = os.environ["static"] + "/static" + COMPRESS_ENABLED = True + COMPRESS_OFFLINE = True + COMPRESS_ROOT = os.environ["static"] + "/compressor-cache" + # So we don't need postgres dependencies + DATABASES = {} + ''; + in '' + mkdir $static + cat weblate/settings_example.py ${staticSettings} > weblate/settings_static.py + export DJANGO_SETTINGS_MODULE="weblate.settings_static" + ${python3.pythonOnBuildForHost.interpreter} manage.py collectstatic --no-input + ${python3.pythonOnBuildForHost.interpreter} manage.py compress + ''; + + passthru = {inherit GI_TYPELIB_PATH;}; + meta = with lib; { description = "Web based translation tool with tight version control integration"; homepage = "https://weblate.org/"; diff --git a/projects/Weblate/examples/base.nix b/projects/Weblate/examples/base.nix index 9008c114..3cf737ec 100644 --- a/projects/Weblate/examples/base.nix +++ b/projects/Weblate/examples/base.nix @@ -11,8 +11,10 @@ # `weblate-generate-secret-key > django-secret` when run as the weblate user. djangoSecretKeyFile = "/var/lib/weblate/django-secret"; smtp = { - # either use smtp.createLocally or specify a valid account on your mail provider. + enable = true; + # Specify a valid account and server for your mail provider. user = "weblate@example.org"; + host = "mail.example.org"; # Manually deployed secret passwordFile = "/var/lib/weblate/smtp-password"; }; diff --git a/projects/Weblate/service.nix b/projects/Weblate/service.nix index 03fad40c..6b0ec09e 100644 --- a/projects/Weblate/service.nix +++ b/projects/Weblate/service.nix @@ -6,125 +6,120 @@ }: let cfg = config.services.weblate; + dataDir = "/var/lib/weblate"; + settingsDir = "${dataDir}/settings"; + + finalPackage = cfg.package.overridePythonAttrs (old: { + # Use a settings module in dataDir, to avoid having to rebuild the package + # when user changes settings. + makeWrapperArgs = + (old.makeWrapperArgs or []) + ++ [ + "--set PYTHONPATH \"${settingsDir}\"" + "--set DJANGO_SETTINGS_MODULE \"settings\"" + ]; + }); + inherit (finalPackage) python; + + pythonEnv = python.buildEnv.override { + extraLibs = with python.pkgs; [ + (toPythonModule finalPackage) + celery + ]; + }; + # This extends and overrides the weblate/settings_example.py code found in upstream. - weblateConfig = '' - - # This was autogenerated by the NixOS module. - - SITE_TITLE = "Weblate" - SITE_DOMAIN = "${cfg.localDomain}" - # TLS terminates at the reverse proxy, but this setting controls how links to weblate are generated. - ENABLE_HTTPS = True - DATA_DIR = "/var/lib/weblate" - STATIC_ROOT = "${pkgs.weblate}/lib/${pkgs.python3.libPrefix}/site-packages/weblate/static/" - MEDIA_ROOT = "/var/lib/weblate/media" - COMPRESS_ROOT = "/var/lib/weblate/compressor-cache/" - DEBUG = False - - DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "HOST": "/run/postgresql", - "NAME": "weblate", - "USER": "weblate", - "PASSWORD": "", - "PORT": "" + weblateConfig = + '' + # This was autogenerated by the NixOS module. + + SITE_TITLE = "Weblate" + SITE_DOMAIN = "${cfg.localDomain}" + # TLS terminates at the reverse proxy, but this setting controls how links to weblate are generated. + ENABLE_HTTPS = True + SESSION_COOKIE_SECURE = ENABLE_HTTPS + DATA_DIR = "${dataDir}" + CACHE_DIR = f"{DATA_DIR}/cache" + STATIC_ROOT = "${finalPackage.static}/static" + MEDIA_ROOT = "/var/lib/weblate/media" + COMPRESS_ROOT = "${finalPackage.static}/compressor-cache" + DEBUG = False + + DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": "/run/postgresql", + "NAME": "weblate", + "USER": "weblate", + } } - } - with open("${cfg.djangoSecretKeyFile}") as f: - SECRET_KEY = f.read().rstrip("\n") - - CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "unix://${config.services.redis.servers.weblate.unixSocket}", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "PASSWORD": None, - "CONNECTION_POOL_KWARGS": {}, + with open("${cfg.djangoSecretKeyFile}") as f: + SECRET_KEY = f.read().rstrip("\n") + + CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "unix://${config.services.redis.servers.weblate.unixSocket}", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "PASSWORD": None, + "CONNECTION_POOL_KWARGS": {}, + }, + "KEY_PREFIX": "weblate", + "TIMEOUT": 3600, }, - "KEY_PREFIX": "weblate", - }, - "avatar": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/var/lib/weblate/avatar-cache", - "TIMEOUT": 86400, - "OPTIONS": {"MAX_ENTRIES": 1000}, + "avatar": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/var/lib/weblate/avatar-cache", + "TIMEOUT": 86400, + "OPTIONS": {"MAX_ENTRIES": 1000}, + } } + + + CELERY_TASK_ALWAYS_EAGER = False + CELERY_BROKER_URL = "redis+socket://${config.services.redis.servers.weblate.unixSocket}" + CELERY_RESULT_BACKEND = CELERY_BROKER_URL + + VCS_BACKENDS = ("weblate.vcs.git.GitRepository",) + + '' + + lib.optionalString cfg.smtp.enable '' + ADMINS = (("Weblate Admin", "${cfg.smtp.user}"),) + + EMAIL_HOST = "${cfg.smtp.host}" + EMAIL_USE_TLS = True + EMAIL_HOST_USER = "${cfg.smtp.user}" + SERVER_EMAIL = "${cfg.smtp.user}" + DEFAULT_FROM_EMAIL = "${cfg.smtp.user}" + EMAIL_PORT = 587 + with open("${cfg.smtp.passwordFile}") as f: + EMAIL_HOST_PASSWORD = f.read().rstrip("\n") + + '' + + cfg.extraConfig; + settings_py = + pkgs.runCommand "weblate_settings.py" + { + inherit weblateConfig; + passAsFile = ["weblateConfig"]; } + '' + mkdir -p $out + cat \ + ${finalPackage}/${python.sitePackages}/weblate/settings_example.py \ + $weblateConfigPath \ + > $out/settings.py + ''; - ADMINS = (("Weblate Admin", "${cfg.smtp.user}"),) - - EMAIL_HOST = "${cfg.smtp.host}" - EMAIL_USE_TLS = True - EMAIL_HOST_USER = "${cfg.smtp.user}" - SERVER_EMAIL = "${cfg.smtp.user}" - DEFAULT_FROM_EMAIL = "${cfg.smtp.user}" - EMAIL_PORT = 587 - with open("${cfg.smtp.passwordFile}") as f: - EMAIL_HOST_PASSWORD = f.read().rstrip("\n") - - CELERY_TASK_ALWAYS_EAGER = False - CELERY_BROKER_URL = "redis+socket://${config.services.redis.servers.weblate.unixSocket}" - CELERY_RESULT_BACKEND = CELERY_BROKER_URL - - ${cfg.extraConfig} - ''; - settings_py = pkgs.runCommand "weblate_settings.py" {} '' - mkdir -p $out - cat ${pkgs.weblate}/lib/${pkgs.python3.libPrefix}/site-packages/weblate/settings_example.py > $out/settings.py - cat >> $out/settings.py < cfg.smtp.host == "127.0.0.1"; - message = ''services.weblate.smtp.host should be "127.0.0.1" if you want to to use services.weblate.smtp.createLocally.''; - } - { - assertion = builtins.compareVersions config.services.postgresql.package.version "15.0" == -1; - message = "Weblate doesn't work with PostgreSQL 15 and higher right now (currently ${config.services.postgresql.package.version}). This is a bug in the NixOS module, so feel free to open a PR."; - } - ]; + systemd.tmpfiles.rules = ["L+ ${settingsDir} - - - - ${settings_py}"]; services.nginx = { enable = true; @@ -211,65 +197,54 @@ in { enableACME = true; locations = { - "= /favicon.ico".alias = "${pkgs.weblate}/lib/${pkgs.python3.libPrefix}/site-packages/weblate/static/favicon.ico"; - "/static/".alias = "${pkgs.weblate}/lib/${pkgs.python3.libPrefix}/site-packages/weblate/static/"; - "/static/CACHE/".alias = "/var/lib/weblate/compressor-cache/CACHE/"; + "= /favicon.ico".alias = "${finalPackage}/${python.sitePackages}/weblate/static/favicon.ico"; + "/static/".alias = "${finalPackage.static}/static/"; + "/static/CACHE/".alias = "${finalPackage.static}/compressor-cache/CACHE/"; "/media/".alias = "/var/lib/weblate/media/"; - "/".extraConfig = '' - # Needed for long running operations in admin interface - uwsgi_read_timeout 3600; - # Adjust based to uwsgi configuration: - uwsgi_pass unix:///run/weblate.socket; - # uwsgi_pass 127.0.0.1:8080; - ''; + "/".proxyPass = "http://unix:///run/weblate.socket"; }; }; }; systemd.services.weblate-postgresql-setup = { description = "Weblate PostgreSQL setup"; - wantedBy = ["multi-user.target"]; after = ["postgresql.service"]; serviceConfig = { Type = "oneshot"; User = "postgres"; Group = "postgres"; ExecStart = '' - ${pkgs.postgresql}/bin/psql weblate -c "CREATE EXTENSION IF NOT EXISTS pg_trgm" + ${config.services.postgresql.package}/bin/psql weblate -c "CREATE EXTENSION IF NOT EXISTS pg_trgm" ''; }; }; systemd.services.weblate-migrate = { description = "Weblate migration"; - wantedBy = [ - "weblate.service" - "multi-user.target" - ]; - after = [ - "postgresql.service" - "weblate-postgresql-setup.service" - ]; + after = ["weblate-postgresql-setup.service"]; + requires = ["weblate-postgresql-setup.service"]; + # We want this to be active on boot, not just on socket activation + wantedBy = ["multi-user.target"]; inherit environment; path = weblatePath; serviceConfig = { Type = "oneshot"; - # WorkingDirectory = pkgs.weblate; StateDirectory = "weblate"; User = "weblate"; Group = "weblate"; - ExecStart = "${pkgs.weblate}/bin/weblate migrate --noinput"; + ExecStart = "${finalPackage}/bin/weblate migrate --noinput"; }; }; systemd.services.weblate-celery = { description = "Weblate Celery"; - wantedBy = ["multi-user.target"]; after = [ "network.target" "redis.service" "postgresql.service" ]; + # We want this to be active on boot, not just on socket activation + wantedBy = ["multi-user.target"]; environment = environment // { @@ -280,14 +255,14 @@ in { # https://github.com/WeblateOrg/weblate/blob/main/weblate/examples/celery-weblate.service serviceConfig = let # We have to push %n through systemd's replacement, therefore %%n. - pidFile = "/run/celery/%%n.pid"; + pidFile = "/run/celery/weblate-%%n.pid"; nodes = "celery notify memory backup translate"; cmd = verb: '' - ${pkgs.weblate.dependencyEnv}/bin/celery multi ${verb} \ + ${pythonEnv}/bin/celery multi ${verb} \ ${nodes} \ -A "weblate.utils" \ --pidfile=${pidFile} \ - --logfile=/var/log/celery/%%n%%I.log \ + --logfile=/var/log/celery/weblate-%%n%%I.log \ --loglevel=DEBUG \ --beat:celery \ --queues:celery=celery \ @@ -306,14 +281,14 @@ in { Type = "forking"; User = "weblate"; Group = "weblate"; - WorkingDirectory = "${pkgs.weblate}/lib/${pkgs.python3.libPrefix}/site-packages/weblate/"; + WorkingDirectory = "${finalPackage}/${python.sitePackages}/weblate/"; RuntimeDirectory = "celery"; RuntimeDirectoryPreserve = "restart"; LogsDirectory = "celery"; ExecStart = cmd "start"; ExecReload = cmd "restart"; ExecStop = '' - ${pkgs.weblate.dependencyEnv}/bin/celery multi stopwait \ + ${pythonEnv}/bin/celery multi stopwait \ ${nodes} \ --pidfile=${pidFile} ''; @@ -322,17 +297,14 @@ in { }; systemd.services.weblate = { - description = "Weblate uWSGI app"; + description = "Weblate Gunicorn app"; after = [ "network.target" - "postgresql.service" - "redis.service" "weblate-migrate.service" - "weblate-postgresql-setup.service" + "weblate-celery.service" ]; requires = [ "weblate-migrate.service" - "weblate-postgresql-setup.service" "weblate-celery.service" "weblate.socket" ]; @@ -342,12 +314,20 @@ in { Type = "notify"; NotifyAccess = "all"; ExecStart = let - uwsgi = pkgs.uwsgi.override {plugins = ["python3"];}; - jsonConfig = pkgs.writeText "uwsgi.json" (builtins.toJSON uwsgiConfig); - in "${uwsgi}/bin/uwsgi --json ${jsonConfig}"; - Restart = "on-failure"; - KillSignal = "SIGTERM"; - WorkingDirectory = pkgs.weblate; + gunicorn = python.pkgs.gunicorn.overridePythonAttrs (old: { + # Allows Gunicorn to set a meaningful process name + dependencies = (old.dependencies or []) ++ old.optional-dependencies.setproctitle; + }); + in '' + ${gunicorn}/bin/gunicorn \ + --name=weblate \ + --bind='unix:///run/weblate.socket' \ + weblate.wsgi + ''; + ExecReload = "kill -s HUP $MAINPID"; + KillMode = "mixed"; + PrivateTmp = true; + WorkingDirectory = dataDir; StateDirectory = "weblate"; RuntimeDirectory = "weblate"; User = "weblate"; @@ -366,10 +346,6 @@ in { }; }; - services.postfix = lib.mkIf cfg.smtp.createLocally { - enable = true; - }; - services.redis.servers.weblate = { enable = true; user = "weblate"; @@ -391,16 +367,9 @@ in { users.users.weblate = { isSystemUser = true; group = "weblate"; - packages = [weblate-env pkgs.weblate] ++ weblatePath; - - # FIXME This is only here because Weblate wants to save something in this dir at runtime... - createHome = true; - home = "/home/weblate"; + packages = [finalPackage] ++ weblatePath; }; - # TODO remove - environment.systemPackages = config.users.users.weblate.packages; - users.groups.weblate.members = [config.services.nginx.user]; }; diff --git a/projects/Weblate/tests/integration-test.nix b/projects/Weblate/tests/integration-test.nix index 9c5fbb77..fd5b2135 100644 --- a/projects/Weblate/tests/integration-test.nix +++ b/projects/Weblate/tests/integration-test.nix @@ -18,26 +18,22 @@ in { meta.maintainers = with pkgs.lib.maintainers; [erictapen]; nodes.server = { - config, pkgs, lib, ... }: { virtualisation.memorySize = 2048; - services.postgresql.package = pkgs.postgresql_14; - imports = [sources.modules."services.weblate"]; services.weblate = { enable = true; localDomain = "${serverDomain}"; - djangoSecretKeyFile = pkgs.writeText "weblate-django-secret" "thisissnakeoilsecret"; - smtp = { - createLocally = true; - user = "weblate@${serverDomain}"; - passwordFile = pkgs.writeText "weblate-smtp-pass" "thisissnakeoilpassword"; - }; + djangoSecretKeyFile = pkgs.writeText "weblate-django-secret" "thisissnakeoilsecretwithmorethan50characterscorrecthorsebatterystaple"; + extraConfig = '' + # Weblate tries to fetch Avatars from the network + ENABLE_AVATARS = False + ''; }; services.nginx.virtualHosts."${serverDomain}" = { @@ -46,24 +42,14 @@ in { sslCertificateKey = certs."${serverDomain}".key; }; - services.postfix = { - enableSubmission = true; - enableSubmissions = true; - submissionsOptions = { - smtpd_sasl_auth_enable = "yes"; - smtpd_client_restrictions = "permit"; - }; - # sslKey = certs.${serverDomain}.key; - # sslCert = certs.${serverDomain}.cert; - }; - security.pki.certificateFiles = [certs.ca.cert]; networking.hosts."::1" = ["${serverDomain}"]; - networking.firewall.allowedTCPPorts = [80 443]; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; - # We need weblate-env available to the root user. - environment.systemPackages = config.users.users.weblate.packages; users.users.weblate.shell = pkgs.bashInteractive; }; @@ -91,7 +77,7 @@ in { start_all() server.wait_for_unit("weblate.socket") server.wait_until_succeeds("curl -f https://${serverDomain}/") - server.succeed("sudo -iu weblate -- weblate-env weblate createadmin --username ${admin.username} --password ${admin.password} --email weblate@example.org") + server.succeed("sudo -iu weblate -- weblate createadmin --username ${admin.username} --password ${admin.password} --email weblate@example.org") # It's easier to replace the generated API token with a predefined one than # to extract it at runtime. @@ -100,7 +86,7 @@ in { client.wait_for_unit("multi-user.target") # Test the official Weblate client wlc. - # client.wait_until_succeeds("wlc --debug list-projects") + client.wait_until_succeeds("REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt wlc --debug list-projects") def call_wl_api(arg): (rv, result) = client.execute("curl -H \"Content-Type: application/json\" -H \"Authorization: Token ${apiToken}\" https://${serverDomain}/api/{}".format(arg)) @@ -114,10 +100,10 @@ in { "email": "test1@example.org" }))) + # TODO: Check sending and receiving email. # server.wait_for_unit("postfix.service") - # The goal is for this to succeed, but there are still some checks failing. - # server.succeed("sudo -iu weblate -- weblate-env weblate check --deploy") - + # TODO: The goal is for this to succeed, but there are still some checks failing. + # server.succeed("sudo -iu weblate -- weblate check --deploy") ''; }