fix views analytics (#885)

* fix views analytics * update ip stripping * update clickhouse tables * fix broken queries * Fix panics * fix download undercounting * fix packerator failing sometimes * run prep
modrinth · Mar 2, 2024 · e2ffeab · e2ffeab
1 parent 04d8341
commit e2ffeab
Show file tree

Hide file tree

Showing 12 changed files with 207 additions and 140 deletions.
diff --git a/.sqlx/query-7ab21e7613dd88e97cf602e76bff62170c13ceef8104a4ce4cb2d101f8ce4f48.json b/.sqlx/query-7ab21e7613dd88e97cf602e76bff62170c13ceef8104a4ce4cb2d101f8ce4f48.json
diff --git a/.sqlx/query-b993ec7579f06603a2a308dccd1ea1fbffd94286db48bc0e36a30f4f6a9d39af.json b/.sqlx/query-b993ec7579f06603a2a308dccd1ea1fbffd94286db48bc0e36a30f4f6a9d39af.json
diff --git a/.sqlx/query-d08c9ef6a8829ce1d23d66f27c58f4b9b64f4ce985e60ded871d1f31eb0c818b.json b/.sqlx/query-d08c9ef6a8829ce1d23d66f27c58f4b9b64f4ce985e60ded871d1f31eb0c818b.json
diff --git a/...588b79a7787b9b3cbbd4f8968cd0d99ed49d.json → ...daf8f4efbe628ee2e8ddbbb4b9734cc9c915.json b/...588b79a7787b9b3cbbd4f8968cd0d99ed49d.json → ...daf8f4efbe628ee2e8ddbbb4b9734cc9c915.json
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/clickhouse/mod.rs b/src/clickhouse/mod.rs
@@ -42,14 +42,15 @@ pub async fn init_client_with_database(
 
                 user_id UInt64,
                 project_id UInt64,
+                monetized Bool DEFAULT True,
 
                 ip IPv6,
                 country String,
                 user_agent String,
-                headers Array(Tuple(String, String)),
+                headers Array(Tuple(String, String))
             )
             ENGINE = MergeTree()
-            PRIMARY KEY (project_id, recorded)
+            PRIMARY KEY (project_id, recorded, ip)
             "
         ))
         .execute()
@@ -71,10 +72,10 @@ pub async fn init_client_with_database(
                 ip IPv6,
                 country String,
                 user_agent String,
-                headers Array(Tuple(String, String)),
+                headers Array(Tuple(String, String))
             )
             ENGINE = MergeTree()
-            PRIMARY KEY (project_id, recorded)
+            PRIMARY KEY (project_id, recorded, ip)
             "
         ))
         .execute()
@@ -94,10 +95,10 @@ pub async fn init_client_with_database(
 
                 loader String,
                 game_version String,
-                parent UInt64,
+                parent UInt64
             )
             ENGINE = MergeTree()
-            PRIMARY KEY (project_id, recorded)
+            PRIMARY KEY (project_id, recorded, user_id)
             "
         ))
         .execute()

diff --git a/src/lib.rs b/src/lib.rs
@@ -16,7 +16,7 @@ use util::cors::default_cors;
 
 use crate::queue::moderation::AutomatedModerationQueue;
 use crate::{
-    // queue::payouts::process_payout,
+    queue::payouts::process_payout,
     search::indexing::index_projects,
     util::env::{parse_strings_from_var, parse_var},
 };
@@ -214,25 +214,25 @@ pub fn app_setup(
         });
     }
 
-    // {
-    //     let pool_ref = pool.clone();
-    //     let redis_ref = redis_pool.clone();
-    //     let client_ref = clickhouse.clone();
-    //     scheduler.run(std::time::Duration::from_secs(60 * 60 * 6), move || {
-    //         let pool_ref = pool_ref.clone();
-    //         let redis_ref = redis_ref.clone();
-    //         let client_ref = client_ref.clone();
-    //
-    //         async move {
-    //             info!("Started running payouts");
-    //             let result = process_payout(&pool_ref, &redis_ref, &client_ref).await;
-    //             if let Err(e) = result {
-    //                 warn!("Payouts run failed: {:?}", e);
-    //             }
-    //             info!("Done running payouts");
-    //         }
-    //     });
-    // }
+    {
+        let pool_ref = pool.clone();
+        let redis_ref = redis_pool.clone();
+        let client_ref = clickhouse.clone();
+        scheduler.run(std::time::Duration::from_secs(60 * 60 * 6), move || {
+            let pool_ref = pool_ref.clone();
+            let redis_ref = redis_ref.clone();
+            let client_ref = client_ref.clone();
+
+            async move {
+                info!("Started running payouts");
+                let result = process_payout(&pool_ref, &redis_ref, &client_ref).await;
+                if let Err(e) = result {
+                    warn!("Payouts run failed: {:?}", e);
+                }
+                info!("Done running payouts");
+            }
+        });
+    }
 
     let ip_salt = Pepper {
         pepper: models::ids::Base62Id(models::ids::random_base62(11)).to_string(),

diff --git a/src/models/v3/analytics.rs b/src/models/v3/analytics.rs
@@ -34,6 +34,8 @@ pub struct PageView {
     pub user_id: u64,
     // Modrinth Project ID (used for payouts)
     pub project_id: u64,
+    // whether this view will be monetized / counted for payouts
+    pub monetized: bool,
 
     // The below information is used exclusively for data aggregation and fraud detection
     // (ex: page view botting).