fix config.toml allowed mime types

scripts/upload_archive_zip.mts

@@ -21,9 +21,10 @@ const __dirname = path.dirname(__filename)
 // Load environment variables
 dotenv.config({ path: path.resolve(__dirname, '../.env.local') })
 
-const isProd = process.argv[3]
-  ? process.argv[3] === 'true'
-  : process.env.NODE_ENV === 'production'
+// const isProd = process.argv[3]
+//   ? process.argv[3] === 'true'
+//   : process.env.NODE_ENV === 'production'
+const isProd = true
 
 const supabaseUrl = isProd
   ? process.env.NEXT_PUBLIC_SUPABASE_URL
@@ -48,13 +49,12 @@ const requiredFiles = [
   'profile',
   'account',
   'tweets',
-  'community-tweet',
   'like',
   'follower',
   'following',
 ]
 
-const optionalFiles = ['note-tweet']
+const optionalFiles = ['note-tweet', 'community-tweet']
 
 const extractZip = async (filePath: string): Promise<Archive> => {
   const directory = await unzipper.Open.file(filePath)

sql/functions/rls/01_apply_public_rls_policies.sql

@@ -22,12 +22,7 @@ BEGIN
     EXECUTE format('
         CREATE POLICY "Data is publicly visible unless marked private" ON %I.%I
         FOR SELECT
-        USING (
-            (SELECT COALESCE(au.keep_private, false) 
-             FROM archive_upload au 
-             WHERE au.id = %I.id) = false
-            OR account_id = (SELECT auth.jwt() -> ''app_metadata'' ->> ''provider_id'')
-        )', schema_name, table_name, table_name);
+        USING (true)', schema_name, table_name);
 
     -- The modification policy remains unchanged
     EXECUTE format('

sql/functions/rls/02_apply_public_entities_rls_policies.sql

@@ -21,18 +21,7 @@ BEGIN
     EXECUTE format('
         CREATE POLICY "Entities are publicly visible unless marked private" ON %I.%I
         FOR SELECT
-        USING (
-            (SELECT COALESCE(au.keep_private, false) 
-             FROM archive_upload au 
-             JOIN public.tweets t ON t.archive_upload_id = au.id
-             WHERE t.tweet_id = %I.tweet_id) = false
-            OR EXISTS (
-                SELECT 1
-                FROM public.tweets t
-                WHERE t.tweet_id = %I.tweet_id
-                AND t.account_id = (SELECT auth.jwt() -> ''app_metadata'' ->> ''provider_id'')
-            )
-        )', schema_name, table_name, table_name, table_name);
+        USING (true)', schema_name, table_name, table_name);
 
     EXECUTE format('
         CREATE POLICY "Entities are modifiable by their users" ON %I.%I TO authenticated

sql/tables/02_archive_upload.sql

@@ -16,3 +16,7 @@ CREATE TABLE IF NOT EXISTS public.archive_upload (
 );
 
 ALTER TABLE public.archive_upload ADD COLUMN upload_phase upload_phase_enum DEFAULT 'uploading';
+
+
+CREATE INDEX "idx_archive_upload_account_id" ON "public"."archive_upload" USING "btree" ("account_id");
+

sql/tables/03_profile.sql

@@ -3,11 +3,16 @@ CREATE TABLE IF NOT EXISTS public.profile (
     account_id TEXT NOT NULL, 
     bio TEXT,
     website TEXT,
-    LOCATION TEXT,
+    location TEXT,
     avatar_media_url TEXT,
     header_media_url TEXT,
     archive_upload_id BIGINT NOT NULL,
     UNIQUE (account_id, archive_upload_id),
     FOREIGN KEY (archive_upload_id) REFERENCES public.archive_upload (id),
     FOREIGN KEY (account_id) REFERENCES public.account (account_id)
-);
+);
+
+
+CREATE INDEX "idx_profile_account_id" ON "public"."profile" USING "btree" ("account_id");
+
+CREATE INDEX "idx_profile_archive_upload_id" ON "public"."profile" USING "btree" ("archive_upload_id");

sql/tables/04_tweets.sql

@@ -16,4 +16,16 @@ CREATE TABLE IF NOT EXISTS public.tweets (
 
 ALTER TABLE public.tweets DROP COLUMN IF EXISTS fts;
 ALTER TABLE public.tweets ADD COLUMN fts tsvector GENERATED ALWAYS AS (to_tsvector('english', full_text)) STORED;
-CREATE INDEX IF NOT EXISTS text_fts ON public.tweets USING gin (fts);
+CREATE INDEX IF NOT EXISTS text_fts ON public.tweets USING gin (fts);
+
+
+CREATE INDEX "idx_tweets_account_id" ON "public"."tweets" USING "btree" ("account_id");
+
+CREATE INDEX "idx_tweets_archive_upload_id" ON "public"."tweets" USING "btree" ("archive_upload_id");
+
+CREATE INDEX "idx_tweets_created_at" ON "public"."tweets" USING "btree" ("created_at" DESC);
+
+
+CREATE INDEX IF NOT EXISTS idx_tweets_reply_to_user_id ON public.tweets USING btree (reply_to_user_id) TABLESPACE pg_default;
+
+CREATE INDEX idx_favorite_count ON tweets (favorite_count);

sql/tables/06_user_mentions.sql

@@ -5,4 +5,10 @@ CREATE TABLE IF NOT EXISTS public.user_mentions (
     FOREIGN KEY (tweet_id) REFERENCES public.tweets (tweet_id),
     FOREIGN KEY (mentioned_user_id) REFERENCES public.mentioned_users (user_id),
     UNIQUE(mentioned_user_id, tweet_id)
-);
+);
+
+
+CREATE INDEX "idx_user_mentions_mentioned_user_id" ON "public"."user_mentions" USING "btree" ("mentioned_user_id");
+
+CREATE INDEX "idx_user_mentions_tweet_id" ON "public"."user_mentions" USING "btree" ("tweet_id");
+

sql/tables/07_tweet_urls.sql

@@ -6,4 +6,6 @@ CREATE TABLE IF NOT EXISTS public.tweet_urls (
     tweet_id TEXT NOT NULL,
     FOREIGN KEY (tweet_id) REFERENCES public.tweets (tweet_id),
     UNIQUE(tweet_id, url)
-);
+);
+
+CREATE INDEX "idx_tweet_urls_tweet_id" ON "public"."tweet_urls" USING "btree" ("tweet_id");

sql/tables/08_tweet_media.sql

@@ -8,4 +8,9 @@ CREATE TABLE IF NOT EXISTS public.tweet_media (
     archive_upload_id BIGINT NOT NULL,
     FOREIGN KEY (archive_upload_id) REFERENCES public.archive_upload (id),
     FOREIGN KEY (tweet_id) REFERENCES public.tweets (tweet_id)
-);
+);
+
+CREATE INDEX "idx_tweet_media_archive_upload_id" ON "public"."tweet_media" USING "btree" ("archive_upload_id");
+
+CREATE INDEX "idx_tweet_media_tweet_id" ON "public"."tweet_media" USING "btree" ("tweet_id");
+

sql/tables/09_followers.sql

@@ -6,4 +6,8 @@ CREATE TABLE IF NOT EXISTS public.followers (
     UNIQUE (account_id, follower_account_id),
     FOREIGN KEY (account_id) REFERENCES public.account (account_id),
     FOREIGN KEY (archive_upload_id) REFERENCES public.archive_upload (id)
-);
+);
+
+CREATE INDEX "idx_followers_account_id" ON "public"."followers" USING "btree" ("account_id");
+
+CREATE INDEX "idx_followers_archive_upload_id" ON "public"."followers" USING "btree" ("archive_upload_id");

sql/tables/10_following.sql

@@ -6,4 +6,9 @@ CREATE TABLE IF NOT EXISTS public.following (
     UNIQUE (account_id, following_account_id),
     FOREIGN KEY (account_id) REFERENCES public.account (account_id),
     FOREIGN KEY (archive_upload_id) REFERENCES public.archive_upload (id)
-);
+);
+
+
+CREATE INDEX "idx_following_account_id" ON "public"."following" USING "btree" ("account_id");
+
+CREATE INDEX "idx_following_archive_upload_id" ON "public"."following" USING "btree" ("archive_upload_id");

sql/tables/12_likes.sql

@@ -7,4 +7,11 @@ CREATE TABLE IF NOT EXISTS public.likes (
     FOREIGN KEY (account_id) REFERENCES public.account (account_id),
     FOREIGN KEY (liked_tweet_id) REFERENCES public.liked_tweets (tweet_id),
     FOREIGN KEY (archive_upload_id) REFERENCES public.archive_upload (id)
-);
+);
+
+
+CREATE INDEX "idx_likes_account_id" ON "public"."likes" USING "btree" ("account_id");
+
+CREATE INDEX "idx_likes_archive_upload_id" ON "public"."likes" USING "btree" ("archive_upload_id");
+
+CREATE INDEX "idx_likes_liked_tweet_id" ON "public"."likes" USING "btree" ("liked_tweet_id");

sql/tables/13_conversations.sql

@@ -2,4 +2,6 @@ CREATE TABLE IF NOT EXISTS "public"."conversations" (
     "tweet_id" text NOT NULL PRIMARY KEY,
     "conversation_id" text,
     FOREIGN KEY (tweet_id) REFERENCES public.tweets(tweet_id)
-);
+);
+
+CREATE INDEX idx_conversation_id ON public.conversations(conversation_id);

sql/triggers/01_activity_summary_job_queue.sql → sql/triggers/01_job_queue.sql

@@ -4,7 +4,7 @@ DROP FUNCTION IF EXISTS refresh_account_activity_summary() CASCADE;
 DROP TRIGGER IF EXISTS update_account_activity_summary ON public.archive_upload;
 DROP TRIGGER IF EXISTS queue_job_on_upload_complete ON public.archive_upload;
 DROP TRIGGER IF EXISTS queue_job_on_upload_delete ON public.archive_upload;
-DROP FUNCTION IF EXISTS private.queue_refresh_activity_summary_on_upload_complete() CASCADE;
+DROP FUNCTION IF EXISTS private.queue_archive_changes_on_upload_complete() CASCADE;
 DROP FUNCTION IF EXISTS private.process_jobs() CASCADE;
 
 
@@ -16,12 +16,12 @@ status TEXT CHECK (status IN ('QUEUED', 'PROCESSING', 'DONE'))
 
 CREATE INDEX IF NOT EXISTS idx_job_queue_status_timestamp ON private.job_queue (status, timestamp);
 
-CREATE OR REPLACE FUNCTION private.queue_refresh_activity_summary()
+CREATE OR REPLACE FUNCTION private.queue_archive_changes()
 RETURNS TRIGGER AS $$
 BEGIN
-RAISE NOTICE 'queue_refresh_activity_summary:Queueing job: refresh_activity_summary';
+RAISE NOTICE 'queue_archive_changes:Queueing job: archive_changes';
 INSERT INTO private.job_queue (key, status)
-VALUES ('refresh_activity_summary', 'QUEUED')
+VALUES ('archive_changes', 'QUEUED')
 ON CONFLICT (key) DO UPDATE
 SET timestamp = CURRENT_TIMESTAMP,
     status = 'QUEUED';
@@ -34,12 +34,12 @@ CREATE TRIGGER queue_job_on_upload_complete
 AFTER UPDATE OF upload_phase ON public.archive_upload
 FOR EACH ROW
 WHEN (NEW.upload_phase = 'completed')
-EXECUTE FUNCTION private.queue_refresh_activity_summary();
+EXECUTE FUNCTION private.queue_archive_changes();
 
 CREATE TRIGGER queue_job_on_upload_delete
 AFTER DELETE ON public.archive_upload
 FOR EACH ROW
-EXECUTE FUNCTION private.queue_refresh_activity_summary();
+EXECUTE FUNCTION private.queue_archive_changes();
 
 CREATE OR REPLACE FUNCTION private.process_jobs()
 RETURNS void AS $$
@@ -70,28 +70,40 @@ SET status = 'PROCESSING'
 WHERE key = v_job.key;
 
 -- Do the job
-IF v_job.key = 'refresh_activity_summary' THEN
-    RAISE NOTICE 'Refreshing materialized views';
+IF v_job.key = 'archive_changes' THEN
+    RAISE NOTICE 'Refreshing materialized views concurrently';
     REFRESH MATERIALIZED VIEW CONCURRENTLY public.global_activity_summary;
     REFRESH MATERIALIZED VIEW CONCURRENTLY public.account_activity_summary;
+    REFRESH MATERIALIZED VIEW CONCURRENTLY public.quote_tweets_view;
+    PERFORM public.post_upload_update_conversation_ids();
 END IF;
 
-
 IF v_job.key = 'update_conversation_ids' THEN
     RAISE NOTICE 'Updating conversation ids';
-    PERFORM private.post_upload_update_conversation_ids();
+
 END IF;
 
 -- Delete the job
 DELETE FROM private.job_queue WHERE key = v_job.key;
 RAISE NOTICE 'Job completed and removed from queue: %', v_job.key;
 END;
 $$ LANGUAGE plpgsql;
-
 -- Enable pg_cron extension if not already enabled
 CREATE EXTENSION IF NOT EXISTS pg_cron;
-
+DO $$
+DECLARE
+    job_id bigint;
+BEGIN
+    FOR job_id IN 
+        SELECT jobid 
+        FROM cron.job 
+        WHERE command LIKE '%SELECT private.process_jobs();%'
+    LOOP
+        PERFORM cron.unschedule(job_id);
+        RAISE NOTICE 'Unscheduled job with ID: %', job_id;
+    END LOOP;
+END $$;
 -- Schedule job to run every minute
 SELECT cron.schedule('* * * * *', $$
 SELECT private.process_jobs();
-$$);
+$$);

sql/views/01_tweet_replies_view.sql

@@ -0,0 +1,10 @@
+CREATE VIEW
+  public.tweet_replies_view AS
+SELECT
+  reply_to_tweet_id,
+  reply_to_user_id
+FROM
+  public.tweets
+WHERE
+  reply_to_tweet_id IS NOT NULL;
+

sql/views/02_tweets_enriched.sql

@@ -0,0 +1,27 @@
+CREATE OR REPLACE VIEW public.tweets_enriched AS
+SELECT 
+    t.tweet_id,
+    t.account_id,
+    a.username,
+    a.account_display_name,
+    t.created_at,
+    t.full_text,
+    t.retweet_count,
+    t.favorite_count,
+    t.reply_to_tweet_id,
+    t.reply_to_user_id,
+    t.reply_to_username,
+    -- Get quoted tweet info
+    qt.quoted_tweet_id,
+    c.conversation_id,
+    -- Get latest avatar URL from profile
+    (SELECT p.avatar_media_url
+     FROM profile p 
+     WHERE p.account_id = t.account_id
+     ORDER BY p.archive_upload_id DESC 
+     LIMIT 1) as avatar_media_url,
+    t.archive_upload_id
+FROM tweets t
+JOIN account a ON t.account_id = a.account_id
+LEFT JOIN conversations c ON t.tweet_id = c.tweet_id
+LEFT JOIN quote_tweets qt ON t.tweet_id = qt.tweet_id; 

sql/views/03_quote_tweets.sql

@@ -0,0 +1,17 @@
+CREATE OR REPLACE MATERIALIZED VIEW
+  public.quote_tweets AS
+SELECT
+  t.tweet_id AS TWEET_ID,
+  SUBSTRING(
+    tu.expanded_url
+    FROM
+      'status/([0-9]+)'
+  ) AS QUOTED_TWEET_ID
+FROM
+  public.tweet_urls tu
+  JOIN public.tweets t ON tu.tweet_id = t.tweet_id
+WHERE
+  tu.expanded_url LIKE 'https://twitter.com/%/status/%'
+  OR tu.expanded_url LIKE 'https://x.com/%/status/%';
+
+CREATE INDEX IF NOT EXISTS idx_quote_tweets_quoted_tweet_id ON public.quote_tweets (QUOTED_TWEET_ID);