Store events locally via SQLite (#38)

* Add expo-sqlite

* Add DB test and demo

* clean up a require cycle

* and another

* Consolidate auth and authActions - no more require cycles

* clean up RelayTest

* show DatabaseTest

* cleanup

* relaypool exports

* clear nostr hook stuff from useAuthed for now

* useRelayPool in dbtest

* ya

* small button showing number of relays

* Extract RelayIndicator

* shh

* shhh

* createTable calls

* databaseReport on how many rows

* try/catch

* useRelayPool refactor options, allow connectNow

* DatabaseTest connect to relayPool

* Start new simple NostrEvent

* useDatabase return db

* handleEvent just make NostrEvent

* parse events

* 6 channels saved

* initial arc_channel_messages maybe

* saving initial channel messages

* saving kind 40 and 42s

* fix useAuthed and hook that up

* wow that was a waste of time

* ok useInterval works thx

* starting to simplify useRelayPool - need to globalize context via zustand

* fix addOrModifyRelay

* looks unfucked now

* finally connectedRelays looks right

* ensure only create 1 sub array even if multiple hooks

* almost subscribing

* k dupe whatever - we got stuff

* saving a bunch more

* move relay check elsewhere

* addrelay in the addrelay

* yeah looking good

* k much smoother

* connecting in authednavigator and got events

* useGlobalFeed now looks in database

* sort msgs chrono

* fetch notes from localdb on interval

* usermetadata

* add saveUserMeta

* add viewrelays screen from settings

* hilarious hack saved a user

* got all usermetadata ehhahahahah

* simpler getFriendMetadata

* start with a FirstLoad screen eheheheh

* rejigger

* return tablecounts

* set up some stuff

* firstload

* ok

* ok saving a bunch

* no reposts or reactions for now

* use zustand store

* inefficient but works

* initial hydration looks good

* fix user hydration

* add comments to mergeSimilarAndRemoveEmptyFilters

* yea the relaypool filter merger was removing the contact filter

* nocache works - now we are receiving kind0s

* axe getfriendemtadata

* ok finally

* hmm

* users and posts tick up

* properly dedupe

* dedupe the multi adds

* alllmost ready

* simplify useglobalfeed

* consolidate initial subscriptions

* harmonize initial sub syntax

* axe old usenostrhook

* connecting usermetadata yo

* homefeed looks good with user metadata

* sort all dem msgs descending chrono

* firstload cleanup

* cleanup

* makesubs async

* clean up useRelayPool

* ya

* lastfetch table and helpers

* and adding channels

* ok channelmsgs works like that too

* just nav to next only if eheheh

* addUsers use latest metadata i think

* and adduser

* ok looking good

* filter out friends we already got

* saving lastfetched and not querying again

* start cleanup

* cleanup

* fix user metadata

* axe prev profilescreen

* console shh

* axe relaytest

* axe prev databasetestdemo

* refactor useUserMetadataForChannel maybe

* refactor useMessagesForChannel maybe

* fix various type errors down to 0

* axe old loadtestpaint - tests pass

* getting user metadata slowly ehehehe

* fix get msgs

* console cleanup

* niiiiiice

* remove require cycles

* put channelscreen in keyboardavoidingview

* bump version to 0.0.2

* add a few more bitcoiners

* navheader padding fixes

* dont autofocus relay field cuz it focuses when hidden

* createaccountscreen padding stuff

* no underscore in number

* keyboardawarescrollview

* bump ios build num

* hey install the keyboardthing

* limit globalfeed to 50 fornow

* replace old nostr class w direct relayinit until we're authed

* bump android build version

* in a channel, only get the last 3 days of msgs

* ok thats a lot better

* last buildnums

app.json

@@ -2,7 +2,7 @@
   "expo": {
     "name": "Arc",
     "slug": "arc",
-    "version": "0.0.1",
+    "version": "0.0.2",
     "sdkVersion": "47.0.0",
     "jsEngine": "hermes",
     "platforms": ["ios", "android"],
@@ -17,7 +17,7 @@
     "ios": {
       "supportsTablet": true,
       "bundleIdentifier": "arcade.labs.arc",
-      "buildNumber": "2",
+      "buildNumber": "3",
       "config": {
         "usesNonExemptEncryption": false
       }
@@ -27,7 +27,7 @@
         "foregroundImage": "./src/assets/icons/adaptive-icon.png",
         "backgroundColor": "#000000"
       },
-      "versionCode": 4
+      "versionCode": 7
     },
     "extra": {
       "eas": {

package.json

@@ -39,6 +39,7 @@
     "expo-random": "~13.0.0",
     "expo-screen-orientation": "^5.0.1",
     "expo-secure-store": "~12.0.0",
+    "expo-sqlite": "~11.0.0",
     "jest": "^26.6.3",
     "jest-expo": "^47.0.1",
     "nostr-relaypool": "0.2.1",
@@ -47,6 +48,7 @@
     "react-dom": "18.2.0",
     "react-native": "0.70.5",
     "react-native-gesture-handler": "2.8.0",
+    "react-native-keyboard-aware-scroll-view": "0.9.5",
     "react-native-reanimated": "2.12.0",
     "react-native-safe-area-context": "4.4.1",
     "react-native-screens": "3.18.2",

src/lib/constants/dummydata.ts

@@ -1,5 +1,6 @@
-import { bechToHex } from 'lib/nostr'
+import { bechToHex } from 'lib/nostr/bech32'
 
+// some folks from https://bitcoinnostr.com/https://bitcoinnostr.com/
 const demoFriendsany = [
   'npub1qg8j6gdwpxlntlxlkew7eu283wzx7hmj32esch42hntdpqdgrslqv024kw', // adam3us
   '00000000827ffaa94bfea288c3dfce4422c794fbb96625b6b31e9049f729d700', // cameri
@@ -10,6 +11,14 @@ const demoFriendsany = [
   'npub1xtscya34g58tk0z605fvr788k263gsu6cy9x0mhnm87echrgufzsevkk5s', // jb55
   'npub1ahxjq4v0zlvexf7cg8j9stumqp3nrtzqzzqxa7szpmcdgqrcumdq0h5ech', // natbrunell
   'npub1qny3tkh0acurzla8x3zy4nhrjz5zd8l9sy9jys09umwng00manysew95gx', // odell
+
+  '883fea4c071fda4406d2b66be21cb1edaf45a3e058050d6201ecf1d3596bbc39', // Adam Curry
+  'npub1m3xsmhkcmv20wgaywfeelmvxeww4d67d8sct5cvlffzfa3lkxpus7vdwhx', // Alpha Zeta
+  'npub1rtlqca8r6auyaw5n5h3l5422dm4sry5dzfee4696fqe8s6qgudks7djtfs', // AMERICAN HODL
+  '1b11ed41e815234599a52050a6a40c79bdd3bfa3d65e5d4a2c8d626698835d6', // André Neves
+  'npub14hn6p34vegy4ckeklz8jq93mendym9asw8z2ej87x2wuwf8werasc6a32x', // Anil
+  'e9e4276490374a0daf7759fd5f475deff6ffb9b0fc5fa98c902b5f4b2fe3bba2', // Ben Arc
+  'npub1yvthlcvf7gz57c4s2kezt787uhgdftf7lq4l9vvatvkwg3sjrqns2puxwr', // Bitcoin Liotta
 ]
 
 export const demoFriends = demoFriendsany.map(

src/lib/constants/relays.ts

@@ -2,7 +2,7 @@ export const DEFAULT_RELAYS = [
   'wss://relay.nostr.ch',
   'wss://arc1.arcadelabs.co',
   'wss://relay.damus.io',
-  'wss://nostr-relay.wlvs.space',
+  //   'wss://nostr-relay.wlvs.space',
   'wss://nostr.fmt.wiz.biz',
   'wss://relay.nostr.bg',
   'wss://nostr.oxtr.dev',

src/lib/database/createTables.ts

@@ -0,0 +1,93 @@
+import * as SQLite from 'expo-sqlite'
+
+export const createTables = async (db: SQLite.WebSQLDatabase) => {
+  try {
+    db.transaction(async (tx) => {
+      for (const createTableCall of createTableCalls) {
+        tx.executeSql(createTableCall)
+      }
+      console.log('Database initialized.')
+    })
+  } catch (error) {
+    console.error('Error creating tables', error)
+  }
+}
+
+const createTableCalls = [
+  `CREATE TABLE IF NOT EXISTS arc_channels(
+    id TEXT PRIMARY KEY NOT NULL,
+    pubkey TEXT NOT NULL,
+    sig TEXT NOT NULL,
+    name TEXT NOT NULL,
+    about TEXT NOT NULL,
+    picture TEXT NOT NULL,
+    created_at INT NOT NULL
+  );`,
+  `CREATE TABLE IF NOT EXISTS arc_channel_messages(
+    id TEXT PRIMARY KEY NOT NULL,
+    content TEXT NOT NULL,
+    created_at INT NOT NULL,
+    kind INT NOT NULL,
+    pubkey TEXT NOT NULL,
+    sig TEXT NOT NULL,
+    tags TEXT NOT NULL,
+    channel_id TEXT NOT NULL,
+    reply_event_id TEXT
+  );`,
+  `CREATE TABLE IF NOT EXISTS arc_direct_messages(
+    id TEXT PRIMARY KEY NOT NULL,
+    content TEXT NOT NULL,
+    created_at INT NOT NULL,
+    kind INT NOT NULL,
+    pubkey TEXT NOT NULL,
+    sig TEXT NOT NULL,
+    tags TEXT NOT NULL,
+    conversation_id TEXT NOT NULL,
+    read BOOLEAN DEFAULT FALSE
+  );`,
+  `CREATE TABLE IF NOT EXISTS arc_notes(
+    id TEXT PRIMARY KEY NOT NULL,
+    content TEXT NOT NULL,
+    created_at INT NOT NULL,
+    kind INT NOT NULL,
+    pubkey TEXT NOT NULL,
+    sig TEXT NOT NULL,
+    tags TEXT NOT NULL,
+    main_event_id TEXT,
+    reply_event_id TEXT,
+    user_mentioned BOOLEAN DEFAULT FALSE,
+    seen BOOLEAN DEFAULT FALSE
+  );`,
+  `CREATE TABLE IF NOT EXISTS arc_reactions(
+    id TEXT PRIMARY KEY NOT NULL,
+    content TEXT NOT NULL,
+    created_at INT NOT NULL,
+    kind INT NOT NULL,
+    pubkey TEXT NOT NULL,
+    sig TEXT NOT NULL,
+    tags TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    type INT NOT NULL
+  );`,
+  `CREATE TABLE IF NOT EXISTS arc_relays(
+    url TEXT PRIMARY KEY NOT NULL,
+    pet INTEGER
+  );`,
+  `CREATE TABLE IF NOT EXISTS arc_users(
+    id TEXT PRIMARY KEY NOT NULL,
+    pubkey TEXT NOT NULL,
+    name TEXT,
+    display_name TEXT,
+    picture TEXT,
+    about TEXT,
+    main_relay TEXT,
+    contact BOOLEAN DEFAULT FALSE,
+    follower BOOLEAN DEFAULT FALSE,
+    lnurl TEXT,
+    created_at INT DEFAULT 0
+  );`,
+  `CREATE TABLE IF NOT EXISTS last_fetch (
+    type TEXT PRIMARY KEY NOT NULL UNIQUE,
+    last_fetch INT NOT NULL
+  );`,
+]

src/lib/database/databaseReport.ts

@@ -0,0 +1,45 @@
+import * as SQLite from 'expo-sqlite'
+
+export const databaseReport = async (db: SQLite.WebSQLDatabase) => {
+  const tableCounts: { [table: string]: number } = {}
+  return new Promise((resolve, reject) => {
+    try {
+      db.transaction((tx) => {
+        tx.executeSql(
+          'SELECT name from sqlite_master WHERE type = "table"',
+          [],
+          (_, { rows: { _array } }) => {
+            const tables = _array.map((t) => t.name)
+            tables.forEach((table) => {
+              tx.executeSql(
+                `SELECT COUNT(*) as count FROM ${table}`,
+                [],
+                (_, { rows: { _array } }) => {
+                  const count = _array[0].count
+                  console.log(` ${count} records in ${table}`)
+                  tableCounts[table] = count
+                  if (Object.keys(tableCounts).length === tables.length) {
+                    resolve(tableCounts)
+                  }
+                },
+                (_, error) => {
+                  console.log(`Error querying ${table}`, error)
+                  reject(error)
+                  return false
+                }
+              )
+            })
+          },
+          (_, error) => {
+            console.log('Error querying sqlite_master', error)
+            reject(error)
+            return false
+          }
+        )
+      })
+    } catch (error) {
+      console.log('Error in databaseReport', error)
+      reject(error)
+    }
+  })
+}

src/lib/database/hydrateStoreFromDatabase.ts

@@ -0,0 +1,65 @@
+import { useStore } from 'app/stores'
+import { db } from 'lib/database/useDatabase'
+
+export const hydrateStoreFromDatabase = async () => {
+  const { addUsers, addNotes, addChannels, addChannelMessages } =
+    useStore.getState()
+
+  db.transaction((tx) => {
+    tx.executeSql(
+      'SELECT * FROM arc_users',
+      [],
+      (_, { rows: { _array } }) => {
+        const users = _array.map((user) => {
+          return user
+        })
+        addUsers(users)
+      },
+      (_, error) => {
+        console.log('Error querying arc_users', error)
+        return false
+      }
+    )
+
+    tx.executeSql(
+      'SELECT * FROM arc_notes',
+      [],
+      (_, { rows: { _array } }) => {
+        const notes = _array.map((note) => {
+          return note
+        })
+        addNotes(notes)
+      },
+      (_, error) => {
+        console.log('Error querying arc_notes', error)
+        return false
+      }
+    )
+
+    tx.executeSql(
+      'SELECT * FROM arc_channels',
+      [],
+      (_, { rows: { _array } }) => {
+        const channels = _array.map((channel) => channel)
+        addChannels(channels)
+      },
+      (_, error) => {
+        console.log('Error querying arc_channels', error)
+        return false
+      }
+    )
+
+    tx.executeSql(
+      'SELECT * FROM arc_channel_messages',
+      [],
+      (_, { rows: { _array } }) => {
+        const channelMessages = _array.map((channelMessage) => channelMessage)
+        addChannelMessages(channelMessages)
+      },
+      (_, error) => {
+        console.log('Error querying arc_channel_messages', error)
+        return false
+      }
+    )
+  })
+}

src/lib/database/index.ts

@@ -0,0 +1,3 @@
+export * from './hydrateStoreFromDatabase'
+export * from './lastFetch'
+export * from './useDatabase'

src/lib/database/lastFetch.ts

@@ -0,0 +1,38 @@
+import { db } from 'lib/database/useDatabase'
+
+export const setLastFetch = (type: string, timestamp: number) => {
+  db.transaction((tx) => {
+    tx.executeSql(
+      `INSERT OR REPLACE INTO last_fetch (type, last_fetch) VALUES (?, ?)`,
+      [type, timestamp],
+      (_, result) => {},
+      (_, error) => {
+        console.log('Error setting last fetch timestamp:', error)
+        return false
+      }
+    )
+  })
+}
+
+export const getLastFetch = (type: string) => {
+  return new Promise((resolve) => {
+    db.transaction((tx) => {
+      tx.executeSql(
+        `SELECT last_fetch FROM last_fetch WHERE type = ?`,
+        [type],
+        (_, result) => {
+          if (result.rows.length > 0) {
+            resolve(result.rows.item(0).last_fetch)
+          } else {
+            resolve(undefined)
+          }
+        },
+        (_, error) => {
+          console.log('Error getting last fetch timestamp:', error)
+          resolve(undefined)
+          return false
+        }
+      )
+    })
+  })
+}

src/lib/database/useDatabase.ts

@@ -0,0 +1,19 @@
+import * as SQLite from 'expo-sqlite'
+import { useEffect } from 'react'
+
+import { createTables } from './createTables'
+
+const openDatabase = () => SQLite.openDatabase('arc53.db')
+export const db = openDatabase()
+
+let did = false
+
+export const useDatabase = () => {
+  useEffect(() => {
+    if (did) return
+    createTables(db)
+    did = true
+  }, [])
+
+  return db
+}

src/lib/hooks/index.ts

@@ -1,5 +1,6 @@
 export * from './useAuthed'
 export * from './useGlobalFeed'
+export * from './useInterval'
 export * from './useLongPress'
 export * from './useNostr'
 export * from './useTheme'

src/lib/hooks/useAuthed.ts

@@ -1,12 +1,10 @@
-import { useNostr } from 'lib/hooks'
 import * as storage from 'lib/storage'
 import { useEffect, useState } from 'react'
 import { useStore } from 'stores'
 
 export const useAuthed = () => {
   const privateKey = useStore((s) => s.user.privateKey)
   const publicKey = useStore((s) => s.user.publicKey)
-  const nostr = useNostr()
 
   const [checkedForKeys, setCheckedForKeys] = useState<boolean>(false)
   const authed = checkedForKeys
@@ -31,15 +29,12 @@ export const useAuthed = () => {
       return
     }
     useStore.setState({ user: { privateKey, publicKey, name: 'Test Ostrich' } })
-    nostr?.setKeys(publicKey, privateKey)
     setCheckedForKeys(true)
   }
 
   useEffect(() => {
-    if (!nostr) return
     checkForKeys()
-  }, [nostr])
+  }, [])
 
-  console.log('Returning authed: ', authed)
   return authed
 }

src/lib/hooks/useGlobalFeed.ts

@@ -1,11 +1,8 @@
-import { useStore } from 'stores'
+import { useStore } from 'app/stores'
 
 export const useGlobalFeed = () => {
-  const events = useStore((s) => s.events)
-  const filteredEvents = events
-    .filter((e) => e.kind === 1)
-    .filter((e) => e.content !== '')
-    .sort((a, b) => b.created_at - a.created_at)
-    .filter((e, i, a) => a.findIndex((t) => t.id === e.id) === i)
-  return filteredEvents
+  const notes = useStore((state) => state.notes).sort(
+    (a, b) => b.created_at - a.created_at
+  )
+  return notes
 }