Add batching to reanalyze_all.

It currently times out in staging.

project/thscoreboard/replays/templates/replays/reanalyze_all.html → project/thscoreboard/replays/templates/replays/reanalyze_batch.html

@@ -10,8 +10,15 @@ <h2>Replays to reanalyze:</h2>
             <li>None!</li>
         {% endfor %}
     </ul>
-    <form method="POST">
+    {% if replays %}
+    <form method="POST" action="{% url 'Replays/ReanalyzePagePost' current_token next_token %}">
         {% csrf_token %}
         <input type="submit" />
     </form>
+    {% endif %}
+    {% if more_pages %}
+    <p><a href="{% url 'Replays/ReanalyzeBatch' next_token %}">Continue to next page</a></p>
+    {% else %}
+    <p>No more replays to analyze.</p>
+    {% endif %}
 {% endblock %}

project/thscoreboard/replays/templates/replays/successfully_reanalyzed_all.html → project/thscoreboard/replays/templates/replays/successfully_reanalyzed_batch.html

@@ -10,4 +10,5 @@ <h2>Reanalyzed replays:</h2>
             <li>None!</li>
         {% endfor %}
     </ul>
+    <a href="{% url 'Replays/ReanalyzeBatch' end_token %}">Continue to next page</a>
 {% endblock %}

project/thscoreboard/replays/urls/replay_urls.py

@@ -20,7 +20,21 @@
     path("publish/<str:game_id>", create_replay.publish_replay_no_file),
     path("user/<str:username>", user.user_page, name="user_page"),
     path("user/<str:username>/json", user.user_page_json),
-    path("reanalyze_all", reanalyze_all_replays.reanalyze_all),
+    path(
+        "reanalyze_all",
+        reanalyze_all_replays.batch_reanalyze_preview,
+        name="Replays/ReanalyzeBatch",
+    ),
+    path(
+        "reanalyze_all/<int:pagination_token>",
+        reanalyze_all_replays.batch_reanalyze_preview,
+        name="Replays/ReanalyzeBatch",
+    ),
+    path(
+        "reanalyze_batch/<int:start_token>/<int:end_token>",
+        reanalyze_all_replays.reanalyze_page,
+        name="Replays/ReanalyzePagePost",
+    ),
     path("<str:game_id>", replay_list.game_scoreboard),
     path("<str:game_id>/json", replay_list.game_scoreboard_json),
     path("<str:game_id>/d<int:difficulty>", replay_list.game_scoreboard),

project/thscoreboard/replays/views/reanalyze_all_replays.py

@@ -10,6 +10,12 @@
 from replays import reanalyze_replay
 
 
+# The number of replays to view and reanalyze at once.
+# This must be low enough that the request will never take over 30 seconds,
+# to avoid timeouts.
+_BATCH_SIZE = 5000
+
+
 def _ShortNameForReplay(r):
     return "{id} ({user}, {game})".format(
         id=r.id,
@@ -18,30 +24,32 @@ def _ShortNameForReplay(r):
     )
 
 
-def _select_all_replays_with_files():
-    return models.Replay.objects.select_related("shot", "shot__game").filter(
-        shot__game__has_replays=True
+def _select_next_replays_with_files(pagination_token, end_token=None):
+    q = models.Replay.objects.filter(id__gt=pagination_token)
+    if end_token is not None:
+        q.filter(id__lte=end_token)
+    return (
+        q.select_related("shot", "shot__game")
+        .filter(shot__game__has_replays=True)
+        .order_by("id")[:_BATCH_SIZE]
     )
 
 
-@http_decorators.require_http_methods(["GET", "HEAD", "POST"])
+@http_decorators.require_http_methods(["GET", "HEAD"])
 @auth_decorators.permission_required("staff", raise_exception=True)
-def reanalyze_all(request):
-    if request.method == "POST":
-        return _post_reanalyze_all(request)
-    else:
-        return _get_reanalyze_all(request)
-
-
-def _get_reanalyze_all(request):
-    # This request is likely to be extremely expensive. At some point,
-    # we'll want to adjust it to be cheaper.
+def batch_reanalyze_preview(request, pagination_token: int = 0):
+    """Preview the reanalysis of a page of replays."""
 
     replay_links = []
 
+    # Bind r now so we can use it outside of the for loop.
+    # This way, we can access the last element in the query without having to
+    # conduct additional queries.
+    r = None
+
     # To lessen the big load on the database, this view function is not
     # performed in one transaction.
-    for r in _select_all_replays_with_files():
+    for r in _select_next_replays_with_files(pagination_token):
         # Typically in Django we would simply pass the replay instance to the
         # template engine, and define its rendering in the template. However,
         # since this method could easily return a very large number of replays,
@@ -57,21 +65,47 @@ def _get_reanalyze_all(request):
                 }
             )
 
+    if r:
+        next_token = r.id
+    else:
+        # Since there are no replays, the "next token" is the same as the
+        # current token.
+        next_token = pagination_token
+
+    more_pages = models.Replay.objects.filter(id__gt=next_token).exists()
+
+    context = {
+        "replays": replay_links,
+        "current_token": pagination_token,
+        "next_token": next_token,
+        "more_pages": more_pages,
+    }
+
     return render(
         request,
-        "replays/reanalyze_all.html",
-        {
-            "replays": replay_links,
-        },
+        "replays/reanalyze_batch.html",
+        context,
     )
 
 
-def _post_reanalyze_all(request):
+@http_decorators.require_http_methods(["POST"])
+@auth_decorators.permission_required("staff", raise_exception=True)
+def reanalyze_page(request, start_token: int, end_token: int):
+    """Reanalyze replays, updating their metadata from the replay file.
+
+    Args:
+        start_token: A pagination token; an inclusive lower bound for the set
+            of replays to be reanalyzed.
+        end_token: A pagination token; an exclusive upper bound for the set of
+            replays to be reanalyzed.
+    """
     replay_links = []
 
     # To lessen the big load on the database, the updates are performed in many
     # small transactions, not one big one.
-    for r in _select_all_replays_with_files():
+    for r in _select_next_replays_with_files(
+        pagination_token=start_token, end_token=end_token
+    ):
         if reanalyze_replay.DoesReplayNeedUpdate(r.id):
             with transaction.atomic():
                 # The replay might have been deleted during this method's
@@ -88,5 +122,7 @@ def _post_reanalyze_all(request):
                     )
 
     return render(
-        request, "replays/successfully_reanalyzed_all.html", {"replays": replay_links}
+        request,
+        "replays/successfully_reanalyzed_batch.html",
+        {"replays": replay_links, "end_token": end_token},
     )