insert tour as raw sql

bahnzumberg · Nov 9, 2024 · aa1e85e · aa1e85e
1 parent 2e18269
commit aa1e85e
Show file tree

Hide file tree

Showing 2 changed files with 91 additions and 48 deletions.
diff --git a/database.sql b/database.sql
@@ -46,9 +46,9 @@ CREATE TABLE tour (
       dec boolean DEFAULT false,
       month_order int DEFAULT 12,
       quality_rating integer DEFAULT 5,
-      full_text TEXT,
-	search_column tsvector,
-      ai_search_column vector(384) DEFAULT NULL,
+      -- full_text TEXT,
+	-- search_column tsvector,
+      ai_search_column vector(1024) DEFAULT NULL,
 	max_ele INT default 0,
 	text_lang VARCHAR(2) default 'de',
       PRIMARY KEY (id)
@@ -61,7 +61,8 @@ CREATE INDEX ON tour (month_order);
 CREATE INDEX ON tour (range);
 CREATE INDEX ON tour (traverse);
 CREATE INDEX ON tour (title);
-CREATE INDEX search_column_idx ON tour USING GIN (search_column);
+CREATE INDEX ON tour USING hnsw (ai_search_column vector_l2_ops);
+// CREATE INDEX search_column_idx ON tour USING GIN (search_column);
 
 
 CREATE TABLE tour_inactive (

diff --git a/src/jobs/sync.js b/src/jobs/sync.js
@@ -837,56 +837,98 @@ const calcMonthOrder = (entry) => {
 
 
 const bulk_insert_tours = async (entries) => {
-    let queries = [];
+    let sql_values = '';
 
     for (let i=0; i<entries.length; i++) {
         let entry = entries[i];
-
-        queries.push({
-            id: entry.id,
-            url: entry.url,
-            provider: entry.provider,
-            hashed_url: entry.hashed_url,
-            description: entry.description,
-            image_url: entry.image_url,
-            ascent: entry.ascent,
-            descent: entry.descent,
-            difficulty: entry.difficulty,
-            difficulty_orig: entry.difficulty_orig,
-            duration: entry.duration,
-            distance: entry.distance,
-            title: entry.title,
-            type: entry.typ,
-            country: entry.country,
-            state: entry.state,
-            range_slug: entry.range_slug,
-            range: entry.range_name,
-            season: entry.season,
-            number_of_days: entry.number_of_days,
-            jan: entry.jan,
-            feb: entry.feb,
-            mar: entry.mar,
-            apr: entry.apr,
-            may: entry.may,
-            jun: entry.jun,
-            jul: entry.jul,
-            aug: entry.aug,
-            sep: entry.sep,
-            oct: entry.oct,
-            nov: entry.nov,
-            dec: entry.dec,
-            month_order: calcMonthOrder(entry),
-            traverse: entry.traverse,
-            quality_rating: entry.quality_rating,
-            // full_text: entry.full_text,
-            ai_search_column: entry.ai_search_column,
-            text_lang: entry.text_lang,
-            max_ele: entry.maxele
-        });
+
+        if (i != 0) {
+            sql_values = sql_values + ",";
+        }
+        sql_values = sql_values + "(" +
+                     entry.id + "," + 
+                     "'" + entry.url + "'" + "," +
+                     "'" + entry.provider + "'" + "," +
+                     "'" + entry.hashed_url + "'" + "," +
+                     "'" + entry.description + "'" + "," +
+                     "'" + entry.image_url + "'" + "," +
+                     entry.ascent + "," +
+                     entry.descent + "," +
+                     entry.difficulty + "," +
+                     "'" + entry.difficulty_orig + "'" + "," +
+                     entry.duration + "," +
+                     entry.distance + "," +
+                     "'" + entry.title + "'" + "," +
+                     "'" + entry.typ + "'" + "," +
+                     "'" + entry.country + "'" + "," +
+                     "'" + entry.state + "'" + "," +
+                     "'" + entry.range_slug + "'" + "," +
+                     "'" + entry.range_name + "'" + "," +
+                     "'" + entry.season + "'" + "," +
+                     entry.number_of_days + "," +
+                     entry.jan + "," +
+                     entry.feb + "," +
+                     entry.mar + "," +
+                     entry.apr + "," +
+                     entry.may + "," +
+                     entry.jun + "," +
+                     entry.jul + "," +
+                     entry.aug + "," +
+                     entry.sep + "," +
+                     entry.oct + "," +
+                     entry.nov + "," +
+                     entry.dec + "," +
+                     calcMonthOrder(entry) + "," +
+                     entry.traverse + "," +
+                     entry.quality_rating + "," +
+                     "'" + entry.ai_search_column + "'" + "," +
+                     "'" + entry.text_lang + "'" + "," +
+                     entry.maxele + ")";
     }
 
+    const sql_insert = `INSERT INTO tour (id, 
+                                          url, 
+                                          provider,
+                                          hashed_url,
+                                          description,
+                                          image_url,
+                                          ascent,
+                                          descent,
+                                          difficulty,
+                                          difficulty_orig,
+                                          duration,
+                                          distance,
+                                          title,
+                                          type,
+                                          country,
+                                          state,
+                                          range_slug,
+                                          range,
+                                          season,
+                                          number_of_days,
+                                          jan,
+                                          feb,
+                                          mar,
+                                          apr,
+                                          may,
+                                          jun,
+                                          jul,
+                                          aug,
+                                          sep,
+                                          oct,
+                                          nov,
+                                          dec,
+                                          month_order,
+                                          traverse,
+                                          quality_rating,
+                                          ai_search_column,
+                                          text_lang,
+                                          max_ele)
+                                          VALUES ${sql_values}`
+    console.log(sql_insert)
+
     try {
-        await knex('tour').insert(queries);
+        await knex.raw(sql_insert)
         return true;
     } catch(err){
         console.log('error: ', err)