Add telegram browser

.github/workflows/test_fastapi_server.yml

@@ -86,7 +86,7 @@ jobs:
 
     - name: Run pytest
       working-directory: ${{ env.SUBDIRECTORY }}
-      run: poetry run python -m pytest
+      run: poetry run python -m pytest test --random-order
 
   build_and_deploy_docker_image:
     strategy:

fastapi_server/.env

@@ -12,4 +12,7 @@ MINIO_URL=0.0.0.0:9000
 MINIO_ACCESS_TOKEN=mylocaldevkey
 MINIO_SECRET_KEY=mylocaldevpassword
 
+# Telegram browser
+ALLOWED_TWITCH_USERS_FOR_TELEGRAM_BROWSER=burnysc2 twitch;some_other_user
+
 STAGE=local_dev

fastapi_server/.env.example

@@ -29,4 +29,8 @@ MINIO_URL=minio-url
 MINIO_ACCESS_TOKEN=minio-access-token
 MINIO_SECRET_KEY=minio-secret-key
 
+# Telegram browser
+ALLOWED_TWITCH_USERS_FOR_TELEGRAM_BROWSER=some;user;and;another
+MINIO_TELEGRAM_FILES_BUCKET=telegram-files-bucket
+
 STAGE=dev

fastapi_server/prisma/migrations/20241022032505_init/migration.sql

@@ -0,0 +1,3 @@
+-- Create extension for index:
+-- @@index([message_text(ops: raw("gin_trgm_ops"))], type: Gin)
+CREATE EXTENSION IF NOT EXISTS pg_trgm;

fastapi_server/prisma/migrations/20241022032647_create_tables/migration.sql

@@ -0,0 +1,141 @@
+-- CreateEnum
+CREATE TYPE "Status" AS ENUM ('NoFile', 'HasFile', 'Queued', 'Downloading', 'Downloaded');
+
+-- CreateTable
+CREATE TABLE "litestar_audiobook_book" (
+    "id" SERIAL NOT NULL,
+    "uploaded_by" TEXT NOT NULL,
+    "book_title" TEXT NOT NULL,
+    "book_author" TEXT NOT NULL,
+    "chapter_count" INTEGER NOT NULL,
+    "upload_date" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "custom_book_title" TEXT,
+    "custom_book_author" TEXT,
+
+    CONSTRAINT "litestar_audiobook_book_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "litestar_audiobook_chapter" (
+    "id" SERIAL NOT NULL,
+    "book_id" INTEGER NOT NULL,
+    "queued" TIMESTAMP(3),
+    "started_converting" TIMESTAMP(3),
+    "chapter_title" TEXT NOT NULL,
+    "chapter_number" INTEGER NOT NULL,
+    "word_count" INTEGER NOT NULL,
+    "sentence_count" INTEGER NOT NULL,
+    "content" TEXT NOT NULL,
+    "minio_object_name" TEXT,
+    "audio_settings" JSONB,
+
+    CONSTRAINT "litestar_audiobook_chapter_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "litestar_telegram_channel" (
+    "id" SERIAL NOT NULL,
+    "channel_id" BIGINT NOT NULL,
+    "channel_title" TEXT NOT NULL,
+    "channel_username" TEXT,
+    "creation_date" TIMESTAMP(3) NOT NULL,
+    "participants" BIGINT NOT NULL,
+    "last_parsed" TIMESTAMP(3) NOT NULL DEFAULT '2000-01-01 00:00:00'::timestamp without time zone,
+
+    CONSTRAINT "litestar_telegram_channel_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "litestar_telegram_message" (
+    "id" SERIAL NOT NULL,
+    "channel_id" BIGINT NOT NULL,
+    "message_id" BIGINT NOT NULL,
+    "message_date" TIMESTAMP(3) NOT NULL,
+    "message_text" TEXT,
+    "amount_of_reactions" BIGINT NOT NULL DEFAULT 0,
+    "amount_of_comments" BIGINT NOT NULL DEFAULT 0,
+    "status" "Status" NOT NULL DEFAULT 'NoFile',
+    "file_downloadinfo_id" BIGINT,
+    "file_downloadinfo_access_hash" BIGINT,
+    "file_downloadinfo_file_reference" BYTEA,
+    "downloading_start_time" TIMESTAMP(3),
+    "mime_type" TEXT,
+    "file_extension" TEXT,
+    "file_size_bytes" BIGINT,
+    "file_duration_seconds" DOUBLE PRECISION,
+    "file_height" INTEGER,
+    "file_width" INTEGER,
+    "minio_object_name" TEXT,
+    "downloading_retry_attempt" INTEGER NOT NULL DEFAULT 0,
+
+    CONSTRAINT "litestar_telegram_message_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "litestar_audiobook_book_id_uploaded_by_idx" ON "litestar_audiobook_book"("id", "uploaded_by");
+
+-- CreateIndex
+CREATE INDEX "litestar_audiobook_book_uploaded_by_book_title_book_author_idx" ON "litestar_audiobook_book"("uploaded_by", "book_title", "book_author");
+
+-- CreateIndex
+CREATE INDEX "litestar_audiobook_chapter_book_id_chapter_number_idx" ON "litestar_audiobook_chapter"("book_id", "chapter_number");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "litestar_telegram_channel_channel_id_key" ON "litestar_telegram_channel"("channel_id");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_channel_channel_id_idx" ON "litestar_telegram_channel"("channel_id");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_channel_channel_title_idx" ON "litestar_telegram_channel"("channel_title");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_channel_channel_username_idx" ON "litestar_telegram_channel"("channel_username");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_channel_id_message_id_idx" ON "litestar_telegram_message"("channel_id", "message_id");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_amount_of_comments_idx" ON "litestar_telegram_message"("amount_of_comments" DESC);
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_amount_of_reactions_idx" ON "litestar_telegram_message"("amount_of_reactions" DESC);
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_channel_id_idx" ON "litestar_telegram_message"("channel_id");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_file_duration_seconds_idx" ON "litestar_telegram_message"("file_duration_seconds");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_file_extension_idx" ON "litestar_telegram_message"("file_extension");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_file_height_idx" ON "litestar_telegram_message"("file_height");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_file_size_bytes_idx" ON "litestar_telegram_message"("file_size_bytes");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_file_width_idx" ON "litestar_telegram_message"("file_width");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_message_date_idx" ON "litestar_telegram_message"("message_date" DESC);
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_message_id_idx" ON "litestar_telegram_message"("message_id");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_message_text_idx" ON "litestar_telegram_message" USING GIN ("message_text" gin_trgm_ops);
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_mime_type_idx" ON "litestar_telegram_message"("mime_type");
+
+-- CreateIndex
+CREATE INDEX "litestar_telegram_message_status_idx" ON "litestar_telegram_message"("status");
+
+-- AddForeignKey
+ALTER TABLE "litestar_audiobook_chapter" ADD CONSTRAINT "litestar_audiobook_chapter_book_id_fkey" FOREIGN KEY ("book_id") REFERENCES "litestar_audiobook_book"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "litestar_telegram_message" ADD CONSTRAINT "litestar_telegram_message_channel_id_fkey" FOREIGN KEY ("channel_id") REFERENCES "litestar_telegram_channel"("channel_id") ON DELETE CASCADE ON UPDATE CASCADE;

fastapi_server/prisma/schema.prisma

@@ -1,7 +1,7 @@
 generator client {
   provider             = "prisma-client-py"
-  recursive_type_depth = "5"
   interface            = "asyncio"
+  recursive_type_depth = "5"
 }
 
 datasource db {
@@ -27,7 +27,6 @@ model AudiobookBook {
 
 model AudiobookChapter {
   id                 Int           @id @default(autoincrement())
-  book               AudiobookBook @relation(fields: [book_id], references: [id], onDelete: Cascade)
   book_id            Int
   queued             DateTime?
   started_converting DateTime?
@@ -38,61 +37,72 @@ model AudiobookChapter {
   content            String
   minio_object_name  String?
   audio_settings     Json?
+  book               AudiobookBook @relation(fields: [book_id], references: [id], onDelete: Cascade)
 
   @@index([book_id, chapter_number])
   @@map("litestar_audiobook_chapter")
 }
 
 model TelegramChannel {
-  id                Int               @id @default(autoincrement())
-  channel_id        Int
-  channel_title     String // Custom name of channel
-  channel_username  String? // Lower case, fixed name, may be None
-  date              DateTime // Creation date
-  participants      Int? // Amount of users / subscribers
-  latest_message_id Int? // Parsing messages from first message
-  TelegramMessage   TelegramMessage[]
+  id               Int               @id @default(autoincrement())
+  channel_id       BigInt            @unique
+  channel_title    String
+  channel_username String?
+  creation_date    DateTime
+  participants     BigInt
+  last_parsed      DateTime          @default(dbgenerated("'2000-01-01 00:00:00'::timestamp without time zone"))
+  TelegramMessage  TelegramMessage[]
 
+  @@index([channel_id])
+  @@index([channel_title])
+  @@index([channel_username])
   @@map("litestar_telegram_channel")
 }
 
+model TelegramMessage {
+  id                               Int             @id @default(autoincrement())
+  channel_id                       BigInt
+  message_id                       BigInt
+  message_date                     DateTime
+  message_text                     String?
+  amount_of_reactions              BigInt          @default(0)
+  amount_of_comments               BigInt          @default(0)
+  status                           Status          @default(NoFile)
+  file_downloadinfo_id             BigInt?
+  file_downloadinfo_access_hash    BigInt?
+  file_downloadinfo_file_reference Bytes?
+  downloading_start_time           DateTime?
+  mime_type                        String?
+  file_extension                   String?
+  file_size_bytes                  BigInt?
+  file_duration_seconds            Float?
+  file_height                      Int?
+  file_width                       Int?
+  minio_object_name                String?
+  downloading_retry_attempt        Int             @default(0)
+  channel                          TelegramChannel @relation(fields: [channel_id], references: [channel_id], onDelete: Cascade)
+
+  @@index([channel_id, message_id])
+  @@index([amount_of_comments(sort: Desc)])
+  @@index([amount_of_reactions(sort: Desc)])
+  @@index([channel_id])
+  @@index([file_duration_seconds])
+  @@index([file_extension])
+  @@index([file_height])
+  @@index([file_size_bytes])
+  @@index([file_width])
+  @@index([message_date(sort: Desc)])
+  @@index([message_id])
+  @@index([message_text(ops: raw("gin_trgm_ops"))], type: Gin)
+  @@index([mime_type])
+  @@index([status])
+  @@map("litestar_telegram_message")
+}
+
 enum Status {
   NoFile
   HasFile
   Queued
   Downloading
   Downloaded
 }
-
-model TelegramMessage {
-  id                    Int             @id @default(autoincrement())
-  channel               TelegramChannel @relation(fields: [channel_id], references: [id], onDelete: Cascade)
-  channel_id            Int
-  message_id            Int
-  message_date          DateTime
-  link                  String // Can be generated from {prefix}/channel_username
-  // Total um of reactions to a message
-  amount_of_reactions   Int             @default(0)
-  // Sum of comments
-  amount_of_comments    Int             @default(0)
-  // Only filled if message has attachments
-  status                Status          @default(NoFile)
-  media_type            String? // TODO Video, Audio, Image, Other
-  // Messages with the same attachments tend to have same unique ids
-  file_unique_id        String?
-  file_size_bytes       BigInt?
-  file_duration_seconds Int? // Filled for video and audio
-  file_height           Int? // Filled for video and image
-  file_width            Int? // Filled for video and image
-  // Filled if media has been downloaded
-  minio_object_name     String?
-
-  @@index([channel_id, message_id])
-  // Potential indexes for filtering messages
-  // @@index([message_date])
-  // @@index([amount_of_reactions])
-  // @@index([amount_of_comments])
-  // @@index([file_size_bytes])
-  // @@index([file_duration_seconds])
-  @@map("litestar_telegram_message")
-}