From e38b9590c5aa9a364bbe33499206683cbd0738d8 Mon Sep 17 00:00:00 2001
From: Clark Kampfe <clark.kampfe@gmail.com>
Date: Sun, 18 Feb 2024 11:33:49 -0600
Subject: [PATCH] clean up entry insertion

get rid of nasty string building
---
 src/rss.rs | 99 ++++++------------------------------------------------
 1 file changed, 11 insertions(+), 88 deletions(-)

diff --git a/src/rss.rs b/src/rss.rs
index b304638..3816981 100644
--- a/src/rss.rs
+++ b/src/rss.rs
@@ -477,21 +477,16 @@ fn add_entries_to_feed(
     if !entries.is_empty() {
         let now = Utc::now();
 
-        let columns = [
-            "feed_id",
-            "title",
-            "author",
-            "pub_date",
-            "description",
-            "content",
-            "link",
-            "updated_at",
-        ];
-
-        let mut entries_values = Vec::with_capacity(entries.len() * columns.len());
-
+        let mut insert_statement = tx.prepare(
+            "INSERT INTO entries (feed_id, title, author, pub_date, description, content, link, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+        )?;
+
+        // in most databases, doing this kind of "multiple inserts in a loop" thing would be bad and slow, but it's ok here because:
+        // 1. it is within single a transaction. in SQLite, doing many writes in the same transaction is actually fast
+        // 2. it is with single prepared statement, which further improves its write throughput
+        // see further: https://stackoverflow.com/questions/1711631/improve-insert-per-second-performance-of-sqlite
         for entry in entries {
-            let values = params![
+            insert_statement.execute(params![
                 feed_id,
                 entry.title,
                 entry.author,
@@ -499,63 +494,14 @@ fn add_entries_to_feed(
                 entry.description,
                 entry.content,
                 entry.link,
-                now,
-            ];
-            entries_values.extend_from_slice(values);
+                now
+            ])?;
         }
-
-        let query = build_bulk_insert_query("entries", &columns, entries);
-
-        tx.execute(&query, entries_values.as_slice())?;
     }
 
     Ok(())
 }
 
-fn build_bulk_insert_query<C: AsRef<str>, R>(table: &str, columns: &[C], rows: &[R]) -> String {
-    let idxs = (1..(rows.len() * columns.len() + 1)).collect::<Vec<_>>();
-
-    let values_groups_string = idxs
-        .chunks(columns.len())
-        .map(|chunk| {
-            let values_string = chunk
-                .iter()
-                .map(|i| format!("?{i}"))
-                .collect::<Vec<_>>()
-                .join(", ");
-            ["(", &values_string, ")"].concat()
-        })
-        .collect::<Vec<_>>()
-        .join(", ");
-
-    let columns_strs = columns
-        .iter()
-        .map(|column| column.as_ref())
-        .collect::<Vec<&str>>();
-
-    let columns_joined = columns_strs.join(", ");
-
-    let mut query = String::with_capacity(
-        "INSERT INTO ".len()
-            + table.len()
-            + 1 // '(' is a char
-            + columns_joined.len()
-            + ") ".len()
-            + "VALUES ".len()
-            + values_groups_string.len(),
-    );
-
-    query.push_str("INSERT INTO ");
-    query.push_str(table);
-    query.push('(');
-    query.push_str(&columns_joined);
-    query.push_str(") ");
-    query.push_str("VALUES ");
-    query.push_str(&values_groups_string);
-
-    query
-}
-
 pub fn get_feed(conn: &rusqlite::Connection, feed_id: FeedId) -> Result<Feed> {
     let s = conn.query_row(
         "SELECT id, title, feed_link, link, feed_kind, refreshed_at, inserted_at, updated_at, latest_etag FROM feeds WHERE id=?1",
@@ -863,29 +809,6 @@ mod tests {
         assert_eq!(new_entries.len(), old_entries.len() - 1);
     }
 
-    #[test]
-    fn build_bulk_insert_query() {
-        let entries = vec!["entry1", "entry2"];
-        let query = super::build_bulk_insert_query(
-            "entries",
-            &[
-                "feed_id",
-                "title",
-                "author",
-                "pub_date",
-                "description",
-                "content",
-                "link",
-                "updated_at",
-            ],
-            &entries,
-        );
-        assert_eq!(
-            query,
-            "INSERT INTO entries(feed_id, title, author, pub_date, description, content, link, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8), (?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)"
-        );
-    }
-
     #[test]
     fn works_transactionally() {
         let mut conn = rusqlite::Connection::open_in_memory().unwrap();