diff --git a/.env.example b/.env.example index fb0f7308d1..20ce8240b4 100644 --- a/.env.example +++ b/.env.example @@ -137,3 +137,6 @@ TWO_FACTOR_LOGIN_MAX_SECONDS=60 # and AWS_S3_CUSTOM_DOMAIN (if used) are added by default. # Value should be a comma-separated list of host names. CSP_ADDITIONAL_HOSTS= +# The last number here means "megabytes" +# Increase if users are having trouble uploading BookWyrm export files. +DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 100) \ No newline at end of file diff --git a/bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py b/bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py new file mode 100644 index 0000000000..ec5b411e2f --- /dev/null +++ b/bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-01-16 10:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0191_merge_20240102_0326"), + ] + + operations = [ + migrations.AddField( + model_name="sitesettings", + name="user_exports_enabled", + field=models.BooleanField(default=False), + ), + ] diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index bd53f1f072..8075b64345 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -96,6 +96,7 @@ class SiteSettings(SiteModel): imports_enabled = models.BooleanField(default=True) import_size_limit = models.IntegerField(default=0) import_limit_reset = models.IntegerField(default=0) + user_exports_enabled = models.BooleanField(default=False) user_import_time_limit = models.IntegerField(default=48) field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"]) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index fcc91857af..cc941da849 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -442,3 +442,5 @@ # Do not change this setting unless you already have an existing # user with the same username - in which case you should change it! INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" + +DATA_UPLOAD_MAX_MEMORY_SIZE = env.int("DATA_UPLOAD_MAX_MEMORY_SIZE", (1024**2 * 100)) diff --git a/bookwyrm/templates/moved.html b/bookwyrm/templates/moved.html index 545fc3d872..382b752be4 100644 --- a/bookwyrm/templates/moved.html +++ b/bookwyrm/templates/moved.html @@ -23,7 +23,7 @@

- {% id_to_username request.user.moved_to as username %} + {% id_to_username request.user.moved_to as username %} {% blocktrans trimmed with moved_to=user.moved_to %} You have moved your account to {{ username }} {% endblocktrans %} diff --git a/bookwyrm/templates/notifications/items/move_user.html b/bookwyrm/templates/notifications/items/move_user.html index b94d96dc49..3121d3f45b 100644 --- a/bookwyrm/templates/notifications/items/move_user.html +++ b/bookwyrm/templates/notifications/items/move_user.html @@ -14,7 +14,7 @@ {% block description %} {% if related_user_moved_to %} - {% id_to_username request.user.moved_to as username %} + {% id_to_username related_user_moved_to as username %} {% blocktrans trimmed %} {{ related_user }} has moved to {{ username }} {% endblocktrans %} diff --git a/bookwyrm/templates/preferences/export-user.html b/bookwyrm/templates/preferences/export-user.html index a468c3f740..cd3119e3e7 100644 --- a/bookwyrm/templates/preferences/export-user.html +++ b/bookwyrm/templates/preferences/export-user.html @@ -46,7 +46,11 @@

Your file will not include:

{% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set the account you are moving to as an alias of this one, or move this account to the new account, before you import your user data." %} {% endspaceless %}

- {% if next_available %} + {% if not site.user_exports_enabled %} +

+ {% trans "New user exports are currently disabled." %} +

+ {% elif next_available %}

{% blocktrans trimmed %} You will be able to create a new export file at {{ next_available }} diff --git a/bookwyrm/templates/settings/imports/imports.html b/bookwyrm/templates/settings/imports/imports.html index 8898aab714..11b3c7e033 100644 --- a/bookwyrm/templates/settings/imports/imports.html +++ b/bookwyrm/templates/settings/imports/imports.html @@ -90,6 +90,33 @@

+ + {% if site.user_exports_enabled %} +
+ + + {% trans "Disable starting new user exports" %} + + + +
+
+ {% trans "This is only intended to be used when things have gone very wrong with exports and you need to pause the feature while addressing issues." %} + {% trans "While exports are disabled, users will not be allowed to start new user exports, but existing exports will not be affected." %} +
+ {% csrf_token %} +
+ +
+
+
@@ -108,7 +135,7 @@ {% trans "Set the value to 0 to not enforce any limit." %}
- + {% csrf_token %} @@ -120,6 +147,28 @@
+ {% else %} +
+
+

{% trans "Users are currently unable to start new user exports. This is the default setting." %}

+ {% if use_s3 %} +

{% trans "It is not currently possible to provide user exports when using s3 storage. The BookWyrm development team are working on a fix for this." %}

+ {% endif %} +
+ {% csrf_token %} +
+ +
+
+ {% endif %}

{% trans "Book Imports" %}

diff --git a/bookwyrm/templatetags/utilities.py b/bookwyrm/templatetags/utilities.py index fca66688ac..230db366e3 100644 --- a/bookwyrm/templatetags/utilities.py +++ b/bookwyrm/templatetags/utilities.py @@ -125,7 +125,8 @@ def id_to_username(user_id): name = parts[-1] value = f"{name}@{domain}" - return value + return value + return "a new user account" @register.filter(name="get_file_size") diff --git a/bookwyrm/tests/views/preferences/test_export.py b/bookwyrm/tests/views/preferences/test_export.py index 4f498f5897..3f758b2f75 100644 --- a/bookwyrm/tests/views/preferences/test_export.py +++ b/bookwyrm/tests/views/preferences/test_export.py @@ -18,7 +18,9 @@ class ExportViews(TestCase): """viewing and creating statuses""" @classmethod - def setUpTestData(self): # pylint: disable=bad-classmethod-argument + def setUpTestData( + self, + ): # pylint: disable=bad-classmethod-argument, disable=invalid-name """we need basic test data and mocks""" with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" @@ -40,6 +42,7 @@ def setUpTestData(self): # pylint: disable=bad-classmethod-argument bnf_id="beep", ) + # pylint: disable=invalid-name def setUp(self): """individual test setup""" self.factory = RequestFactory() @@ -53,11 +56,12 @@ def tst_export_get(self, *_): def test_export_file(self, *_): """simple export""" - models.ShelfBook.objects.create( + shelfbook = models.ShelfBook.objects.create( shelf=self.local_user.shelf_set.first(), user=self.local_user, book=self.book, ) + book_date = str.encode(f"{shelfbook.shelved_date.date()}") request = self.factory.post("") request.user = self.local_user export = views.Export.as_view()(request) @@ -66,7 +70,7 @@ def test_export_file(self, *_): # pylint: disable=line-too-long self.assertEqual( export.content, - b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,rating,review_name,review_cw,review_content\r\nTest Book,," - + self.book.remote_id.encode("utf-8") - + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,\r\n", + b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date\r\n" + + b"Test Book,,%b,,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,,,to-read,To Read,%b\r\n" + % (self.book.remote_id.encode("utf-8"), book_date), ) diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 76e60245bc..1a577c84bc 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -338,6 +338,16 @@ views.disable_imports, name="settings-imports-disable", ), + re_path( + r"^settings/user-exports/enable/?$", + views.enable_user_exports, + name="settings-user-exports-enable", + ), + re_path( + r"^settings/user-exports/disable/?$", + views.disable_user_exports, + name="settings-user-exports-disable", + ), re_path( r"^settings/imports/enable/?$", views.enable_imports, diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index 3be813208f..f11c11dd68 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -18,6 +18,8 @@ set_import_size_limit, set_user_import_completed, set_user_import_limit, + enable_user_exports, + disable_user_exports, ) from .admin.ip_blocklist import IPBlocklist from .admin.invite import ManageInvites, Invite, InviteRequest diff --git a/bookwyrm/views/admin/imports.py b/bookwyrm/views/admin/imports.py index a85d6c79e5..0924536bfa 100644 --- a/bookwyrm/views/admin/imports.py +++ b/bookwyrm/views/admin/imports.py @@ -9,7 +9,7 @@ from bookwyrm import models from bookwyrm.views.helpers import redirect_to_referer -from bookwyrm.settings import PAGE_LENGTH +from bookwyrm.settings import PAGE_LENGTH, USE_S3 # pylint: disable=no-self-use @@ -59,6 +59,7 @@ def get(self, request, status="active"): "import_size_limit": site_settings.import_size_limit, "import_limit_reset": site_settings.import_limit_reset, "user_import_time_limit": site_settings.user_import_time_limit, + "use_s3": USE_S3, } return TemplateResponse(request, "settings/imports/imports.html", data) @@ -126,3 +127,25 @@ def set_user_import_limit(request): site.user_import_time_limit = int(request.POST.get("limit")) site.save(update_fields=["user_import_time_limit"]) return redirect("settings-imports") + + +@require_POST +@permission_required("bookwyrm.edit_instance_settings", raise_exception=True) +# pylint: disable=unused-argument +def enable_user_exports(request): + """Allow users to export account data""" + site = models.SiteSettings.objects.get() + site.user_exports_enabled = True + site.save(update_fields=["user_exports_enabled"]) + return redirect("settings-imports") + + +@require_POST +@permission_required("bookwyrm.edit_instance_settings", raise_exception=True) +# pylint: disable=unused-argument +def disable_user_exports(request): + """Don't allow users to export account data""" + site = models.SiteSettings.objects.get() + site.user_exports_enabled = False + site.save(update_fields=["user_exports_enabled"]) + return redirect("settings-imports") diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index f54d97ccb1..d16f3aaa38 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -17,7 +17,8 @@ from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from bookwyrm.settings import PAGE_LENGTH -# pylint: disable=no-self-use + +# pylint: disable=no-self-use,too-many-locals @method_decorator(login_required, name="dispatch") class Export(View): """Let users export data""" @@ -54,7 +55,19 @@ def post(self, request): fields = ( ["title", "author_text"] + deduplication_fields - + ["rating", "review_name", "review_cw", "review_content"] + + [ + "start_date", + "finish_date", + "stopped_date", + "rating", + "review_name", + "review_cw", + "review_content", + "review_published", + "shelf", + "shelf_name", + "shelf_date", + ] ) writer.writerow(fields) @@ -70,6 +83,24 @@ def post(self, request): book.rating = review_rating.rating if review_rating else None + readthrough = ( + models.ReadThrough.objects.filter(user=request.user, book=book) + .order_by("-start_date", "-finish_date") + .first() + ) + if readthrough: + book.start_date = ( + readthrough.start_date.date() if readthrough.start_date else None + ) + book.finish_date = ( + readthrough.finish_date.date() if readthrough.finish_date else None + ) + book.stopped_date = ( + readthrough.stopped_date.date() + if readthrough.stopped_date + else None + ) + review = ( models.Review.objects.filter( user=request.user, book=book, content__isnull=False @@ -78,9 +109,27 @@ def post(self, request): .first() ) if review: + book.review_published = ( + review.published_date.date() if review.published_date else None + ) book.review_name = review.name book.review_cw = review.content_warning - book.review_content = review.raw_content + book.review_content = ( + review.raw_content if review.raw_content else review.content + ) # GoodReads imported reviews do not have raw_content, but content. + + shelfbook = ( + models.ShelfBook.objects.filter(user=request.user, book=book) + .order_by("-shelved_date", "-created_date", "-updated_date") + .last() + ) + if shelfbook: + book.shelf = shelfbook.shelf.identifier + book.shelf_name = shelfbook.shelf.name + book.shelf_date = ( + shelfbook.shelved_date.date() if shelfbook.shelved_date else None + ) + writer.writerow([getattr(book, field, "") or "" for field in fields]) return HttpResponse( diff --git a/nginx/development b/nginx/development index 841db0124a..64cd1b9119 100644 --- a/nginx/development +++ b/nginx/development @@ -64,13 +64,18 @@ server { # directly serve images and static files from the # bookwyrm filesystem using sendfile. # make the logs quieter by not reporting these requests - location ~ ^/(images|static)/ { + location ~ \.(bmp|ico|jpg|jpeg|png|tif|tiff|webp|css|js)$ { root /app; try_files $uri =404; add_header X-Cache-Status STATIC; access_log off; } + # block access to any non-image files from images or static + location ~ ^/images/ { + return 403; + } + # monitor the celery queues with flower, no caching enabled location /flower/ { proxy_pass http://flower:8888; diff --git a/nginx/production b/nginx/production index 9018ab9de7..76ed19449f 100644 --- a/nginx/production +++ b/nginx/production @@ -96,12 +96,17 @@ server { # # directly serve images and static files from the # # bookwyrm filesystem using sendfile. # # make the logs quieter by not reporting these requests -# location ~ ^/(images|static)/ { +# location ~ \.(bmp|ico|jpg|jpeg|png|tif|tiff|webp|css|js)$ { # root /app; # try_files $uri =404; # add_header X-Cache-Status STATIC; # access_log off; # } + +# # block access to any non-image files from images or static +# location ~ ^/images/ { +# return 403; +# } # # # monitor the celery queues with flower, no caching enabled # location /flower/ {