feat: implement rate limiting and add download progress indicator

package.json

@@ -13,21 +13,21 @@
     "lint:fix": "pnpm --recursive run lint:fix"
   },
   "dependencies": {
-    "@supabase/supabase-js": "^2.48.0",
+    "@supabase/supabase-js": "^2.48.1",
     "date-fns": "^4.1.0",
     "type-fest": "^4.33.0",
     "zod": "3.23.8"
   },
   "devDependencies": {
-    "@antfu/eslint-config": "^3.16.0",
-    "@stylistic/eslint-plugin": "^2.13.0",
-    "@t3-oss/env-core": "^0.11.1",
+    "@antfu/eslint-config": "^4.1.0",
+    "@stylistic/eslint-plugin": "^3.0.0",
+    "@t3-oss/env-core": "0.11.1",
     "@total-typescript/ts-reset": "^0.6.1",
-    "@types/node": "^22.10.7",
-    "eslint": "^9.18.0",
-    "eslint-plugin-package-json": "^0.20.1",
+    "@types/node": "^22.10.10",
+    "eslint": "^9.19.0",
+    "eslint-plugin-package-json": "^0.21.1",
     "jsonc-eslint-parser": "^2.4.0",
-    "openapi-typescript": "^7.5.2",
+    "openapi-typescript": "^7.6.0",
     "pathe": "^2.0.2",
     "prettier": "^3.4.2",
     "taze": "^18.3.0",

ui/auto-imports.d.ts

@@ -95,6 +95,8 @@ declare global {
   const IconCodiconBlank: typeof import('~icons/codicon/blank.jsx')['default']
   const IconCodiconRegex: typeof import('~icons/codicon/regex.jsx')['default']
   const IconEpCircleCloseFilled: typeof import('~icons/ep/circle-close-filled.jsx')['default']
+  const IconF7ArrowDownCircleFill: typeof import('~icons/f7/arrow-down-circle-fill.jsx')['default']
+  const IconF7MultiplyCircle: typeof import('~icons/f7/multiply-circle.jsx')['default']
   const IconFaLanguage: typeof import('~icons/fa/language.jsx')['default']
   const IconGgDarkMode: typeof import('~icons/gg/dark-mode.jsx')['default']
   const IconHeroiconsBars3: typeof import('~icons/heroicons/bars3.jsx')['default']
@@ -118,6 +120,7 @@ declare global {
   const IconMaterialSymbolsRefreshRounded: typeof import('~icons/material-symbols/refresh-rounded.jsx')['default']
   const IconMingcuteHome3Line: typeof import('~icons/mingcute/home3-line.jsx')['default']
   const IconMingcuteUser4Fill: typeof import('~icons/mingcute/user4-fill.jsx')['default']
+  const IconOuiTokenKey: typeof import('~icons/oui/token-key.jsx')['default']
   const IconPhSun: typeof import('~icons/ph/sun.jsx')['default']
   const IconPrimeEllipsisH: typeof import('~icons/prime/ellipsis-h.jsx')['default']
   const IconSolarListCheckBold: typeof import('~icons/solar/list-check-bold.jsx')['default']

ui/package.json

@@ -29,23 +29,23 @@
     "@number-flow/react": "^0.5.5",
     "@radix-ui/colors": "^3.0.0",
     "@radix-ui/react-accordion": "^1.2.2",
-    "@radix-ui/react-alert-dialog": "^1.1.4",
+    "@radix-ui/react-alert-dialog": "^1.1.5",
     "@radix-ui/react-checkbox": "^1.1.3",
     "@radix-ui/react-collapsible": "^1.1.2",
-    "@radix-ui/react-dialog": "^1.1.4",
-    "@radix-ui/react-dropdown-menu": "^2.1.4",
+    "@radix-ui/react-dialog": "^1.1.5",
+    "@radix-ui/react-dropdown-menu": "^2.1.5",
     "@radix-ui/react-icons": "^1.3.2",
     "@radix-ui/react-label": "^2.1.1",
-    "@radix-ui/react-menubar": "^1.1.4",
-    "@radix-ui/react-popover": "^1.1.4",
-    "@radix-ui/react-select": "^2.1.4",
+    "@radix-ui/react-menubar": "^1.1.5",
+    "@radix-ui/react-popover": "^1.1.5",
+    "@radix-ui/react-select": "^2.1.5",
     "@radix-ui/react-separator": "^1.1.1",
     "@radix-ui/react-slot": "^1.1.1",
-    "@radix-ui/react-toast": "^1.2.4",
+    "@radix-ui/react-toast": "^1.2.5",
     "@radix-ui/react-toggle": "^1.1.1",
     "@react-hook/resize-observer": "^2.0.2",
     "@react-hookz/web": "^25.0.1",
-    "@sentry/react": "^8.50.0",
+    "@sentry/react": "^8.51.0",
     "@tanstack/query-async-storage-persister": "^5.64.2",
     "@tanstack/query-core": "^5.64.2",
     "@tanstack/query-sync-storage-persister": "^5.64.2",
@@ -67,18 +67,19 @@
     "figma-squircle": "^1.1.0",
     "file-type": "^20.0.0",
     "foxact": "^0.2.43",
-    "i18next": "^24.2.1",
+    "i18next": "^24.2.2",
     "immer": "^10.1.1",
     "jotai": "^2.11.1",
     "jotai-devtools": "^0.10.1",
-    "jotai-effect": "^1.0.7",
+    "jotai-effect": "^1.1.4",
     "jotai-immer": "^0.4.1",
     "jotai-tanstack-query": "^0.9.0",
     "lodash-es": "^4.17.21",
-    "lucide-react": "^0.473.0",
+    "lucide-react": "^0.474.0",
     "ofetch": "^1.4.1",
     "openapi-fetch": "^0.13.4",
-    "openapi-react-query": "^0.2.9",
+    "openapi-react-query": "^0.3.0",
+    "p-queue": "^8.1.0",
     "pdfjs-dist": "^4.10.38",
     "polished": "^4.3.1",
     "react": "19.0.0",
@@ -104,7 +105,7 @@
     "@eslint-react/eslint-plugin": "^1.24.1",
     "@eslint/compat": "^1.2.5",
     "@iconify-icon/react": "^2.3.0",
-    "@iconify/json": "^2.2.298",
+    "@iconify/json": "^2.2.299",
     "@iconify/react": "^5.2.0",
     "@svgr/core": "^8.1.0",
     "@svgr/plugin-jsx": "^8.1.0",
@@ -116,7 +117,7 @@
     "@types/react": "npm:types-react@rc",
     "@types/react-dom": "npm:types-react-dom@rc",
     "@vitejs/plugin-react": "^4.3.4",
-    "@vitest/ui": "^3.0.3",
+    "@vitest/ui": "^3.0.4",
     "autoprefixer": "^10.4.20",
     "babel-plugin-react-compiler": "19.0.0-beta-e552027-20250112",
     "eslint-plugin-react-compiler": "19.0.0-beta-e552027-20250112",
@@ -126,8 +127,8 @@
     "eslint-plugin-valtio": "^0.8.0",
     "postcss": "^8.5.1",
     "prettier": "^3.4.2",
-    "prettier-plugin-tailwindcss": "^0.6.10",
-    "rollup": "^4.31.0",
+    "prettier-plugin-tailwindcss": "^0.6.11",
+    "rollup": "^4.32.0",
     "rollup-plugin-visualizer": "^5.14.0",
     "supabase": "^2.6.8",
     "tailwindcss": "^3.4.17",
@@ -140,6 +141,6 @@
     "vite-plugin-checker": "^0.8.0",
     "vite-plugin-html": "^3.2.2",
     "vite-plugin-inspect": "^10.1.0",
-    "vitest": "^3.0.3"
+    "vitest": "^3.0.4"
   }
 }

ui/src/api/opensubtitles.ts

@@ -2,11 +2,13 @@ import type { MergeDeep, PartialDeep } from 'type-fest'
 
 import { queryOptions, useMutation, useQuery } from '@tanstack/react-query'
 import { ofetch } from 'ofetch'
+import PQueue from 'p-queue'
 
 import type { paths } from '@/types/schema-opensubtitles'
 
 import { env } from '@/env'
 import { omitUndefined } from '@/lib/utilities'
+import { subtitleDownloadProgressAtom } from '@/store/useVocab'
 
 type subtitles_parameters_query = {
   // https://forum.opensubtitles.org/viewtopic.php?t=17146&start=105#p48222
@@ -101,25 +103,72 @@ export function useOpenSubtitlesLogin() {
   })
 }
 
-type Download = {
+export type Download = {
   Body: NonNullable<paths['/download']['post']['requestBody']>['content']['application/json']
   Response: paths['/download']['post']['responses'][200]['content']['application/json']
 }
 
-export function useOpenSubtitlesDownload() {
+const osQueue = new PQueue({
+  concurrency: 20,
+  interval: 1000,
+  intervalCap: 20,
+  carryoverConcurrencyCount: true,
+})
+
+function useRequestSubtitleURL() {
   const { baseUrl } = useAtomValue(opensubtitlesReqAtom)
   const Authorization = useAtomValue(opensubtitlesAuthorizationAtom)
   return useMutation({
-    mutationKey: ['subtitles-download'],
+    mutationKey: ['requestSubtitleDownloadURL'],
     mutationFn: (body: Download['Body']) => {
-      return ofetch<Download['Response']>(`${baseUrl}/download`, {
+      return osQueue.add(() => ofetch<Download['Response']>(`${baseUrl}/download`, {
         method: 'POST',
         body,
         headers: omitUndefined({
           Authorization,
         }),
-        retry: 3,
+      }), {
+        throwOnTimeout: true,
+      })
+    },
+    retry: 4,
+    retryDelay: (failureCount) => 1000 * (failureCount - 1),
+  })
+}
+
+function useGetFileByLink() {
+  return useMutation({
+    mutationKey: ['getFileByLink'] as const,
+    mutationFn: async (link: string) => {
+      return osQueue.add(() => ofetch<string>(link), {
+        throwOnTimeout: true,
+        priority: 1,
+      })
+    },
+    retry: 4,
+    retryDelay: (failureCount) => 1000 * (failureCount - 1),
+  })
+}
+
+export function useOpenSubtitlesDownload() {
+  const setSubtitleDownloadProgress = useSetAtom(subtitleDownloadProgressAtom)
+  const { mutateAsync: requestSubtitleURL } = useRequestSubtitleURL()
+  const { mutateAsync: getFileByLink } = useGetFileByLink()
+  return useMutation({
+    mutationKey: ['getSubtitleByFileId'] as const,
+    mutationFn: async (body: Download['Body']) => {
+      const file = await requestSubtitleURL(body)
+      return {
+        file,
+        text: await getFileByLink(file.link),
+      }
+    },
+    onSuccess: (data, body) => {
+      setSubtitleDownloadProgress((prev) => {
+        prev.push(body)
       })
     },
+    retry: 4,
+    retryDelay: (failureCount) => 1000 * (failureCount - 1),
   })
 }

ui/src/api/vocab-api.ts

@@ -1,13 +1,16 @@
+import type { RealtimePostgresInsertPayload, RealtimePostgresUpdatePayload } from '@supabase/supabase-js'
 import type { Tables } from '@ui/database.types'
 import type { ValueOf } from 'type-fest'
 
 import { UTCDateMini } from '@date-fns/utc'
-import { REALTIME_CHANNEL_STATES, type RealtimePostgresInsertPayload, type RealtimePostgresUpdatePayload } from '@supabase/supabase-js'
+import { REALTIME_CHANNEL_STATES } from '@supabase/supabase-js'
 import { queryOptions, useMutation, useQuery } from '@tanstack/react-query'
 import { atomWithQuery } from 'jotai-tanstack-query'
 
+import type { LearningPhase, VocabState } from '@/lib/LabeledTire'
+
 import { usePageVisibility } from '@/hooks/utils'
-import { LEARNING_PHASE, type LearningPhase, type VocabState } from '@/lib/LabeledTire'
+import { LEARNING_PHASE } from '@/lib/LabeledTire'
 import { omitUndefined } from '@/lib/utilities'
 import { queryClient, sessionAtom, supabase, vocabRealtimeSyncStatusAtom } from '@/store/useVocab'
 

ui/src/components/VocabData.tsx

@@ -1,3 +1,5 @@
+import type { InitialTableState } from '@tanstack/react-table'
+
 import usePagination from '@mui/material/usePagination'
 import { useUnmountEffect } from '@react-hookz/web'
 import {
@@ -7,11 +9,11 @@ import {
   getFilteredRowModel,
   getPaginationRowModel,
   getSortedRowModel,
-  type InitialTableState,
   useReactTable,
 } from '@tanstack/react-table'
 import { useSessionStorage } from 'react-use'
 
+import type { LearningPhase } from '@/lib/LabeledTire'
 import type { LabelDisplayTable } from '@/lib/vocab'
 
 import { statusLabels, userVocabularyAtom } from '@/api/vocab-api'
@@ -24,7 +26,7 @@ import { useVocabularyCommonColumns } from '@/components/vocabulary/columns'
 import { VocabularyMenu } from '@/components/vocabulary/menu'
 import { useLastTruthy } from '@/lib/hooks'
 import { SortIcon } from '@/lib/icon-utils'
-import { LEARNING_PHASE, type LearningPhase } from '@/lib/LabeledTire'
+import { LEARNING_PHASE } from '@/lib/LabeledTire'
 import { tryGetRegex } from '@/lib/regex'
 import { findClosest, getFallBack } from '@/lib/utilities'
 import { vocabRealtimeSyncStatusAtom } from '@/store/useVocab'

ui/src/components/VocabSource.tsx

@@ -1,3 +1,5 @@
+import type { InitialTableState } from '@tanstack/react-table'
+
 import usePagination from '@mui/material/usePagination'
 import { useUnmountEffect } from '@react-hookz/web'
 import {
@@ -7,14 +9,14 @@ import {
   getFilteredRowModel,
   getPaginationRowModel,
   getSortedRowModel,
-  type InitialTableState,
   useReactTable,
 } from '@tanstack/react-table'
 import {
   useTranslation,
 } from 'react-i18next'
 import { useSessionStorage } from 'react-use'
 
+import type { LearningPhase } from '@/lib/LabeledTire'
 import type { LabelDisplaySource } from '@/lib/vocab'
 
 import { SearchWidget } from '@/components/search-widget'
@@ -27,7 +29,7 @@ import { ExampleSentence } from '@/components/vocabulary/example-sentence'
 import { VocabularyMenu } from '@/components/vocabulary/menu'
 import { useLastTruthy } from '@/lib/hooks'
 import { SortIcon } from '@/lib/icon-utils'
-import { LEARNING_PHASE, type LearningPhase } from '@/lib/LabeledTire'
+import { LEARNING_PHASE } from '@/lib/LabeledTire'
 import { tryGetRegex } from '@/lib/regex'
 import { findClosest } from '@/lib/utilities'
 import { isSourceTextStaleAtom } from '@/store/useVocab'

ui/src/components/subtitle/movie-subtitles.tsx

@@ -1,6 +1,8 @@
+import type { InitialTableState } from '@tanstack/react-table'
+
 import usePagination from '@mui/material/usePagination'
 import NumberFlow from '@number-flow/react'
-import { createColumnHelper, getCoreRowModel, getExpandedRowModel, getFilteredRowModel, getPaginationRowModel, getSortedRowModel, type InitialTableState, useReactTable } from '@tanstack/react-table'
+import { createColumnHelper, getCoreRowModel, getExpandedRowModel, getFilteredRowModel, getPaginationRowModel, getSortedRowModel, useReactTable } from '@tanstack/react-table'
 
 import type { SubtitleResponseData } from '@/api/opensubtitles'
 
@@ -55,10 +57,11 @@ function useMovieColumns<T extends SubtitleData>() {
                 )}
               >
                 <Checkbox
+                  variant="radio"
                   onClick={(e) => e.stopPropagation()}
                   checked={row.getIsSelected()}
                   onCheckedChange={(checked) => {
-                    onRowSelectionChange?.(checked, row)
+                    onRowSelectionChange?.(checked, row, 'singleRow')
                   }}
                 />
                 {'\u200B'}
@@ -135,19 +138,21 @@ export function MovieSubtitleFiles({
         <SquircleMask
           className="flex size-full flex-col bg-[--theme-bg]"
         >
-          <div className="flex h-12 gap-2 p-1.5">
-            <div className="flex aspect-square h-full items-center justify-center">
-              {isPending ? (
-                <IconLucideLoader2
-                  className="animate-spin"
-                />
-              ) : null}
+          <div>
+            <div className="flex h-9 gap-2 p-1.5">
+              <div className="flex aspect-square items-center justify-center">
+                {isPending ? (
+                  <IconLucideLoader2
+                    className="animate-spin"
+                  />
+                ) : null}
+              </div>
             </div>
           </div>
           <div
-            className="size-full grow overflow-auto overflow-y-scroll overscroll-contain"
+            className="grow overflow-auto overflow-y-scroll overscroll-contain"
           >
-            <table className="relative min-w-full border-separate border-spacing-0">
+            <table className="relative border-separate border-spacing-0">
               <TableHeader>
                 {table.getHeaderGroups().map((headerGroup) => (
                   <tr key={headerGroup.id}>
@@ -173,7 +178,7 @@ export function MovieSubtitleFiles({
               </tbody>
             </table>
           </div>
-          <div className="flex w-full flex-wrap items-center justify-between gap-0.5 border-t border-t-zinc-200 py-1 pr-0.5 tabular-nums dark:border-slate-800">
+          <div className="flex flex-wrap items-center justify-between gap-0.5 border-t border-t-zinc-200 py-1 pr-0.5 tabular-nums dark:border-slate-800">
             <TablePagination
               items={items}
               table={table}
@@ -189,7 +194,7 @@ export function MovieSubtitleFiles({
               </div>
             </div>
           </div>
-          <div className="flex w-full justify-center border-t border-solid border-t-zinc-200 bg-background dark:border-slate-800">
+          <div className="flex justify-center border-t border-solid border-t-zinc-200 bg-background dark:border-slate-800">
             <div className="flex h-7 items-center text-xs tabular-nums">
               <span>
                 <NumberFlow