Add stress test

package.json

@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "@chainsafe/libp2p-gossipsub": "^13.1.0",
-    "@orbitdb/core": "https://github.com/orbitdb/orbitdb.git#expose-identities",
+    "@orbitdb/core": "https://github.com/orbitdb/orbitdb.git#fix/manifest-store",
     "@orbitdb/set-db": "^1.0.2",
     "blockstore-level": "^1.1.8",
     "datastore-level": "^10.1.8",

src/utils/libp2p-config.js

@@ -33,7 +33,15 @@ export const config = ({ peerId, port, websocketPort } = {}) => {
     services: {
       identify: identify(),
       pubsub: gossipsub({
-        emitSelf: true
+        emitSelf: true,
+      //   doPX: true,
+      // scoreThresholds: {
+      //   publishThreshold: -1000000000,
+      //   gossipThreshold: -1000000001,
+      //   graylistThreshold: -80000000000,
+      //   acceptPXThreshold: 10,
+      //   opportunisticGraftThreshold: 20
+      // }
       })
     }
   }

test/stress-test.js

@@ -0,0 +1,109 @@
+import { multiaddr } from '@multiformats/multiaddr'
+import { strictEqual, deepStrictEqual } from 'assert'
+import { rimraf } from 'rimraf'
+import { launchLander } from './utils/launch-lander.js'
+import { launchOrbiter } from './utils/launch-orbiter.js'
+import waitFor from './utils/wait-for.js'
+import connectPeers from './utils/connect-nodes-via-relay.js'
+
+const isBrowser = () => typeof window !== 'undefined'
+
+describe('End-to-End Browser Tests', function () {
+  describe('Orbiter in the browser', function () {
+    this.timeout(10000000)
+
+    // const orbiterAddress1 = isBrowser()
+    //   ? multiaddr('/ip4/127.0.0.1/tcp/55441/ws/p2p/16Uiu2HAmBzKcgCfpJ4j4wJSLkKLbCVvnNBWPnhexrnJWJf1fDu5y')
+    //   : multiaddr('/ip4/127.0.0.1/tcp/54321/p2p/16Uiu2HAmBzKcgCfpJ4j4wJSLkKLbCVvnNBWPnhexrnJWJf1fDu5y')
+
+    // const orbiterAddress2 = isBrowser()
+    //   ? multiaddr('/ip4/127.0.0.1/tcp/55442/ws/p2p/16Uiu2HAmATMovCwY46yyJib7bGZF2f2XLRar7d7R3NJCSJtuyQLt')
+    //   : multiaddr('/ip4/127.0.0.1/tcp/54322/p2p/16Uiu2HAmATMovCwY46yyJib7bGZF2f2XLRar7d7R3NJCSJtuyQLt')
+
+    let orbiter
+    let lander1
+    let lander2
+
+    before(async function () {
+      orbiter = await launchOrbiter({ directory: 'orbiter3' })
+    })
+
+    after(async function () {
+      if (orbiter) {
+        await orbiter.shutdown()
+      }
+      await rimraf('./orbiter3')
+    })
+
+    it.only('pin and replicate a database - lander1->orbiter1(nodejs)->orbiter2(nodejs)->lander2', async function () {
+      const rounds = 50
+      const entryAmount = 100
+      const addr = orbiter.orbitdb.ipfs.libp2p.getMultiaddrs().shift()
+
+      console.log("start", addr)
+      for (let k = 1; k <= rounds; k ++) {
+        let replicated = false
+
+        // lander1 = await launchLander({ orbiterAddress: orbiterAddress1, directory: 'lander4' })
+        lander1 = await launchLander({ orbiterAddress: addr, directory: 'lander4' })
+        await orbiter.auth.add(lander1.orbitdb.identity.id)
+
+        const db1 = await lander1.orbitdb.open('my-db3')
+
+        // console.log("--", lander1.orbitdb.ipfs.libp2p.peerId.toString(), db1.address)
+
+        for (let i = 0; i < entryAmount; i++) {
+          await db1.add('hello world ' + i)
+        }
+
+        const expected = await db1.all()
+
+        // console.time('pin')
+        await lander1.pin(db1.address)
+        // console.timeEnd('pin')
+
+        await lander1.shutdown()
+
+        lander2 = await launchLander({ orbiterAddress: addr, directory: 'lander5' })
+        await orbiter.auth.add(lander2.orbitdb.identity.id)
+        // lander2 = await launchLander({ orbiterAddress: orbiterAddress1, directory: 'lander5' })
+
+
+        // console.time('pin2')
+        // await lander2.pin(db1.address)
+        // console.timeEnd('pin2')
+
+        // console.time('replicate')
+        // console.log("open", db1.address)
+        console.time('round ' + k + "/" + rounds)
+        const db2 = await lander2.orbitdb.open(db1.address)
+
+        const onConnected = (peerId, heads) => {
+          replicated = true
+        }
+
+        db2.events.on('join', onConnected)
+
+        await waitFor(() => replicated, () => true)
+        // console.timeEnd('replicate')
+        console.timeEnd('round ' + k + "/" + rounds)
+
+        const res = await db2.all()
+
+        strictEqual(expected.length, entryAmount)
+        strictEqual(res.length, entryAmount)
+        deepStrictEqual(expected, res)
+        // console.log('done  ' + k + "/" + rounds)
+
+        if (lander1) {
+          await lander1.shutdown()
+        }
+        if (lander2) {
+          await lander2.shutdown()
+        }
+        await rimraf('./lander4')
+        await rimraf('./lander5')
+      }
+    })
+  })
+})

test/utils/launch-lander.js

@@ -1,10 +1,13 @@
+import { join } from 'path'
 import { createHelia } from 'helia'
+import { LevelBlockstore } from 'blockstore-level'
+import { LevelDatastore } from 'datastore-level'
 import { createLibp2p } from 'libp2p'
 import { bitswap } from '@helia/block-brokers'
 import { createOrbitDB } from '@orbitdb/core'
 import Lander from '../../src/lib/lander.js'
-// import connectPeers from './connect-nodes.js'
-import connect from './connect-nodes-via-relay.js'
+import connectPeers from './connect-nodes.js'
+// import connect from './connect-nodes-via-relay.js'
 
 import Libp2pOptions from './test-config/lander-libp2p-config.js'
 
@@ -19,22 +22,28 @@ const heliaOptions = {
 export const launchLander = async ({ directory, orbiterAddress } = {}) => {
   const options = Libp2pOptions
 
+  const blockstore = new LevelBlockstore(join(directory, '/', 'ipfs', '/', 'blocks'))
+  const datastore = new LevelDatastore(join(directory, '/', 'ipfs', '/', 'data'))
+
   const libp2p = await createLibp2p({ ...options })
-  const ipfs = await createHelia({ libp2p, ...heliaOptions })
+  // const ipfs = await createHelia({ libp2p, ...heliaOptions })
+  const ipfs = await createHelia({ libp2p, ...heliaOptions, datastore, blockstore })
 
   directory = directory || './lander'
 
   const orbitdb = await createOrbitDB({ ipfs, directory })
 
-  // await connectPeers(ipfs, orbiterAddress)
-  await connect(ipfs, orbiterAddress)
+  await connectPeers(ipfs, orbiterAddress)
+  // await connect(ipfs, orbiterAddress)
 
   const lander = await Lander({ orbitdb, orbiterAddressOrId: orbiterAddress })
 
   // Helper function for tests
   lander.shutdown = async () => {
     await orbitdb.stop()
     await ipfs.stop()
+    await datastore.close()
+    await blockstore.close()
   }
 
   return lander

test/utils/launch-orbiter.js

@@ -35,6 +35,7 @@ export const launchOrbiter = async ({ directory } = {}) => {
   const datastore = new LevelDatastore(join(directory, '/', 'ipfs', '/', 'data'))
 
   const libp2p = await createLibp2p({ ...options })
+  // const ipfs = await createHelia({ libp2p, ...heliaOptions })
   const ipfs = await createHelia({ libp2p, ...heliaOptions, datastore, blockstore })
   const orbitdb = await createOrbitDB({ ipfs, directory, id })
   const orbiter = await Orbiter({ orbitdb })

test/utils/test-config/lander-libp2p-config.js

@@ -1,9 +1,9 @@
 import { identify } from '@libp2p/identify'
 import { noise } from '@chainsafe/libp2p-noise'
 import { yamux } from '@chainsafe/libp2p-yamux'
+import { tcp } from '@libp2p/tcp'
 import { webSockets } from '@libp2p/websockets'
 import { gossipsub } from '@chainsafe/libp2p-gossipsub'
-import { tcp } from '@libp2p/tcp'
 import { mdns } from '@libp2p/mdns'
 
 const Libp2pOptions = {
@@ -27,13 +27,21 @@ const Libp2pOptions = {
     denyDialMultiaddr: () => false // allow dialling of private addresses.
   },
   peerDiscovery: [
-    mdns()
+    // mdns()
   ],
   services: {
     identify: identify(),
     pubsub: gossipsub({
-      emitSelf: true,
-      allowPublishToZeroPeers: true
+      emitSelf: false,
+      allowPublishToZeroTopicPeers: true,
+      // allowPublishToZeroPeers: true,
+      // scoreThresholds: {
+        // publishThreshold: -1000000000,
+        // gossipThreshold: -1000000001,
+        // graylistThreshold: -80000000000,
+        // acceptPXThreshold: 10,
+        // opportunisticGraftThreshold: 20
+      // }
     })
   }
 }

test/utils/test-config/orbiter-libp2p-config.js

@@ -1,9 +1,9 @@
 import { identify } from '@libp2p/identify'
 import { noise } from '@chainsafe/libp2p-noise'
 import { yamux } from '@chainsafe/libp2p-yamux'
+import { tcp } from '@libp2p/tcp'
 import { webSockets } from '@libp2p/websockets'
 import { gossipsub } from '@chainsafe/libp2p-gossipsub'
-import { tcp } from '@libp2p/tcp'
 import { mdns } from '@libp2p/mdns'
 
 const Libp2pOptions = {
@@ -27,12 +27,23 @@ const Libp2pOptions = {
     denyDialMultiaddr: () => false // allow dialling of private addresses.
   },
   peerDiscovery: [
-    mdns()
+    // mdns()
   ],
   services: {
     identify: identify(),
     pubsub: gossipsub({
-      emitSelf: true
+      emitSelf: false,
+      allowPublishToZeroTopicPeers: true,
+      // allowPublishToZeroPeers: true,
+      // emitSelf: true,
+      // doPX: true,
+      scoreThresholds: {
+        // publishThreshold: -1000000000,
+        // gossipThreshold: -1000000001,
+        graylistThreshold: -80000000000,
+        // acceptPXThreshold: 10,
+        // opportunisticGraftThreshold: 20
+      }
     })
   }
 }