Merge pull request #63 from orbitdb/pinner/new-refactor-2-helia-upgrade

Pinner/new refactor 2 helia upgrade

.dockerignore

@@ -0,0 +1,3 @@
+# .dockerignore
+.git
+node_modules

.well-known/funding-manifest-urls

@@ -0,0 +1 @@
+https://orbitdb.org/funding.json

Dockerfile

@@ -0,0 +1,14 @@
+# syntax=docker/dockerfile:1
+
+FROM node:20.17 AS build-env
+ENV NODE_ENV=production
+
+WORKDIR /app
+COPY ["package.json", "package-lock.json*", "./"]
+RUN npm ci --only=production
+COPY . .
+
+FROM node:20.17-slim
+COPY --from=build-env /app /app
+WORKDIR /app
+CMD [ "node", "./src/bin/cli.js", "daemon" ]

package.json

@@ -9,15 +9,6 @@
   "bin": {
     "voyager": "./src/bin/cli.js"
   },
-  "dependencies": {
-    "@chainsafe/libp2p-gossipsub": "^13.1.0",
-    "@orbitdb/core": "https://github.com/orbitdb/orbitdb.git#expose-identities",
-    "@orbitdb/set-db": "^1.0.2",
-    "blockstore-level": "^1.1.8",
-    "datastore-level": "^10.1.8",
-    "helia": "^4.2.5",
-    "yargs": "^17.7.2"
-  },
   "devDependencies": {
     "c8": "^10.1.2",
     "it-drain": "^3.0.7",
@@ -49,5 +40,13 @@
     "env": [
       "mocha"
     ]
+  },
+  "dependencies": {
+    "@chainsafe/libp2p-gossipsub": "^14.1.0",
+    "@orbitdb/core": "^2.4.0",
+    "@orbitdb/set-db": "^1.1.0",
+    "blockstore-level": "^2.0.1",
+    "datastore-level": "^11.0.1",
+    "helia": "^5.1.0"
   }
 }

src/daemon.js

@@ -12,9 +12,7 @@ import { Access } from './lib/authorization.js'
 import { config as libp2pConfig } from './utils/libp2p-config.js'
 import { rpc as rpcId, appPath, rpcPath, app, orbiter as orbiterId, orbiterPath } from './utils/id.js'
 import { saveConfig } from './utils/config-manager.js'
-import { createFromPrivKey } from '@libp2p/peer-id-factory'
 import { logger, enable } from '@libp2p/logger'
-
 export default async ({ options }) => {
   options = options || {}
 
@@ -50,10 +48,10 @@ export default async ({ options }) => {
   let identities = await Identities({ keystore })
   await identities.createIdentity({ id })
 
-  const peerId = await createFromPrivKey(await keystore.getKey(id))
+  const privateKey = await keystore.getKey(id)
   await keystore.close()
 
-  const libp2p = await createLibp2p(await libp2pConfig({ peerId, port: options.port, websocketPort: options.wsport }))
+  const libp2p = await createLibp2p(await libp2pConfig({ privateKey, port: options.port, websocketPort: options.wsport }))
 
   log('peerid:', libp2p.peerId.toString())
   for (const addr of libp2p.getMultiaddrs().map(e => e.toString())) {

src/lib/lander.js

@@ -13,7 +13,7 @@ export default async ({ orbitdb, orbiterAddressOrId }) => {
       })()
     }
 
-    const stream = await orbitdb.ipfs.libp2p.dialProtocol(orbiterAddressOrId, voyagerProtocol, { runOnTransientConnection: true })
+    const stream = await orbitdb.ipfs.libp2p.dialProtocol(orbiterAddressOrId, voyagerProtocol, { runOnLimitedConnection: true })
 
     await pipe(pinDBs, stream, async (source) => {
       for await (const chunk of source) {
@@ -39,7 +39,7 @@ export default async ({ orbitdb, orbiterAddressOrId }) => {
       })()
     }
 
-    const stream = await orbitdb.ipfs.libp2p.dialProtocol(orbiterAddressOrId, voyagerProtocol, { runOnTransientConnection: true })
+    const stream = await orbitdb.ipfs.libp2p.dialProtocol(orbiterAddressOrId, voyagerProtocol, { runOnLimitedConnection: true })
 
     await pipe(unpinDBs, stream, async source => {
       for await (const chunk of source) {

src/lib/orbiter.js

@@ -28,7 +28,7 @@ export default async ({ orbitdb, defaultAccess, verbose } = {}) => {
     await pipe(stream, handleRequest({ log, orbitdb, pins, dbs, auth }), stream)
   }
 
-  await orbitdb.ipfs.libp2p.handle(voyagerProtocol, handleMessages, { runOnTransientConnection: true })
+  await orbitdb.ipfs.libp2p.handle(voyagerProtocol, handleMessages, { runOnLimitedConnection: true })
 
   log('open pinned databases')
 

src/rpc-client.js

@@ -1,11 +1,11 @@
 import { join } from 'path'
 import { Identities, KeyStore } from '@orbitdb/core'
 import { createLibp2p } from 'libp2p'
-import { createFromPrivKey } from '@libp2p/peer-id-factory'
 import { Commands, sendCommand } from './rpc/index.js'
 import { rpc as rpcId, appPath, rpcPath } from './utils/id.js'
 import { loadConfig } from './utils/config-manager.js'
 import { config as libp2pConfig } from './utils/libp2p-config.js'
+import { privateKeyFromProtobuf } from '@libp2p/crypto/keys'
 
 const authAdd = (identity, libp2p, address) => async ({ id }) => {
   return sendCommand(identity, libp2p, address, Commands.AUTH_ADD, [id])
@@ -37,8 +37,8 @@ export default async ({ id, directory }) => {
   const identities = await Identities({ keystore })
   const identity = await identities.createIdentity({ id: rpcId })
 
-  const peerId = await createFromPrivKey(await keystore.getKey(rpcId))
-  const libp2p = await createLibp2p(await libp2pConfig({ peerId }))
+  const privateKey = privateKeyFromProtobuf((await keystore.getKey(id)).bytes)
+  const libp2p = await createLibp2p(await libp2pConfig({ privateKey }))
 
   return {
     authAdd: authAdd(identity, libp2p, config.orbiter.api),

src/utils/libp2p-config.js

@@ -4,9 +4,10 @@ import { yamux } from '@chainsafe/libp2p-yamux'
 import { tcp } from '@libp2p/tcp'
 import { webSockets } from '@libp2p/websockets'
 import { gossipsub } from '@chainsafe/libp2p-gossipsub'
-import { mdns } from '@libp2p/mdns'
+// import { mdns } from '@libp2p/mdns'
+import { bootstrap } from '@libp2p/bootstrap'
 
-export const config = ({ peerId, port, websocketPort } = {}) => {
+export const config = ({ privateKey, port, websocketPort } = {}) => {
   const conf = {
     addresses: {
       listen: [
@@ -18,7 +19,7 @@ export const config = ({ peerId, port, websocketPort } = {}) => {
       tcp(),
       webSockets()
     ],
-    connectionEncryption: [
+    connectionEncrypters: [
       noise()
     ],
     streamMuxers: [
@@ -28,21 +29,24 @@ export const config = ({ peerId, port, websocketPort } = {}) => {
       denyDialMultiaddr: () => false // allow dialling of private addresses.
     },
     peerDiscovery: [
-      mdns()
+      bootstrap({
+        list: ['/ip4/127.0.0.1/tcp/54321/p2p/16Uiu2HAmBzKcgCfpJ4j4wJSLkKLbCVvnNBWPnhexrnJWJf1fDu5y']
+      })
+      /* mdns() */
     ],
     services: {
       identify: identify(),
       pubsub: gossipsub({
         emitSelf: true,
         scoreThresholds: {
-          graylistThreshold: -80000000000,
+          graylistThreshold: -80000000000
         }
       })
     }
   }
 
-  if (peerId) {
-    conf.peerId = peerId
+  if (privateKey) {
+    conf.privateKey = privateKey
   }
 
   return conf

test/e2e-browser.test.js

@@ -108,7 +108,6 @@ describe('End-to-End Browser Tests', function () {
 
       console.time('replicate')
       const db2 = await lander3.orbitdb.open(db1.address)
-
       const onConnected = (peerId, heads) => {
         replicated = true
       }

test/stress-test.js

@@ -1,12 +1,12 @@
-import { multiaddr } from '@multiformats/multiaddr'
+// import { multiaddr } from '@multiformats/multiaddr'
 import { strictEqual, deepStrictEqual } from 'assert'
 import { rimraf } from 'rimraf'
 import { launchLander } from './utils/launch-lander.js'
 import { launchOrbiter } from './utils/launch-orbiter.js'
 import waitFor from './utils/wait-for.js'
-import connectPeers from './utils/connect-nodes-via-relay.js'
+// import connectPeers from './utils/connect-nodes-via-relay.js'
 
-const isBrowser = () => typeof window !== 'undefined'
+// const isBrowser = () => typeof window !== 'undefined'
 
 describe('End-to-End Browser Tests', function () {
   describe('Orbiter in the browser', function () {
@@ -35,13 +35,13 @@ describe('End-to-End Browser Tests', function () {
       await rimraf('./orbiter3')
     })
 
-    it.only('pin and replicate a database - lander1->orbiter1(nodejs)->orbiter2(nodejs)->lander2', async function () {
+    it('pin and replicate a database - lander1->orbiter1(nodejs)->orbiter2(nodejs)->lander2', async function () {
       const rounds = 50
       const entryAmount = 100
       const addr = orbiter.orbitdb.ipfs.libp2p.getMultiaddrs().shift()
 
-      console.log("start", addr)
-      for (let k = 1; k <= rounds; k ++) {
+      console.log('start', addr)
+      for (let k = 1; k <= rounds; k++) {
         let replicated = false
 
         // lander1 = await launchLander({ orbiterAddress: orbiterAddress1, directory: 'lander4' })
@@ -68,14 +68,13 @@ describe('End-to-End Browser Tests', function () {
         await orbiter.auth.add(lander2.orbitdb.identity.id)
         // lander2 = await launchLander({ orbiterAddress: orbiterAddress1, directory: 'lander5' })
 
-
         // console.time('pin2')
         // await lander2.pin(db1.address)
         // console.timeEnd('pin2')
 
         // console.time('replicate')
         // console.log("open", db1.address)
-        console.time('round ' + k + "/" + rounds)
+        console.time('round ' + k + '/' + rounds)
         const db2 = await lander2.orbitdb.open(db1.address)
 
         const onConnected = (peerId, heads) => {
@@ -86,7 +85,7 @@ describe('End-to-End Browser Tests', function () {
 
         await waitFor(() => replicated, () => true)
         // console.timeEnd('replicate')
-        console.timeEnd('round ' + k + "/" + rounds)
+        console.timeEnd('round ' + k + '/' + rounds)
 
         const res = await db2.all()
 

test/utils/connect-nodes-via-relay.js

@@ -21,7 +21,6 @@ const connectIpfsNodes = async (ipfs, peerAddress, options = {
       address1 = ipfs.libp2p.getMultiaddrs().filter(ma => WebRTC.matches(ma)).pop()
       return address1 != null
     }, () => true)
-
     await ipfs.libp2p.dial(peerAddress)
   } else {
     if (peerAddress) {

test/utils/launch-lander.js

@@ -18,7 +18,6 @@ const heliaOptions = {
 
 export const launchLander = async ({ directory, orbiterAddress } = {}) => {
   const options = Libp2pOptions
-
   const libp2p = await createLibp2p({ ...options })
   const ipfs = await createHelia({ libp2p, ...heliaOptions })
 

test/utils/launch-orbiter.js

@@ -28,7 +28,6 @@ export const launchOrbiter = async ({ directory } = {}) => {
   const options = Libp2pOptions
 
   directory = directory || './orbiter'
-
   const id = orbiterId
 
   const blockstore = new LevelBlockstore(join(directory, '/', 'ipfs', '/', 'blocks'))

test/utils/relay.js

@@ -5,21 +5,17 @@ import { circuitRelayServer } from '@libp2p/circuit-relay-v2'
 import { webSockets } from '@libp2p/websockets'
 import * as filters from '@libp2p/websockets/filters'
 import { identify } from '@libp2p/identify'
-import { createFromPrivKey } from '@libp2p/peer-id-factory'
-import { unmarshalPrivateKey } from '@libp2p/crypto/keys'
+import { privateKeyFromProtobuf } from '@libp2p/crypto/keys'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 
 // output of: console.log(server.peerId.privateKey.toString('hex'))
 const relayPrivKey = '08011240821cb6bc3d4547fcccb513e82e4d718089f8a166b23ffcd4a436754b6b0774cf07447d1693cd10ce11ef950d7517bad6e9472b41a927cd17fc3fb23f8c70cd99'
 // the peer id of the above key
 // const relayId = '12D3KooWAJjbRkp8FPF5MKgMU53aUTxWkqvDrs4zc1VMbwRwfsbE'
-
-const encoded = uint8ArrayFromString(relayPrivKey, 'hex')
-const privateKey = await unmarshalPrivateKey(encoded)
-const peerId = await createFromPrivKey(privateKey)
+const privateKey = privateKeyFromProtobuf(uint8ArrayFromString(relayPrivKey, 'hex'))
 
 const server = await createLibp2p({
-  peerId,
+  privateKey,
   addresses: {
     listen: ['/ip4/0.0.0.0/tcp/12345/ws']
   },
@@ -28,17 +24,11 @@ const server = await createLibp2p({
       filter: filters.all
     })
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   services: {
     identify: identify(),
-    relay: circuitRelayServer({
-      reservations: {
-        maxReservations: 5000,
-        reservationTtl: 1000,
-        defaultDataLimit: BigInt(1024 * 1024 * 1024)
-      }
-    })
+    relay: circuitRelayServer()
   }
 })
 

test/utils/test-config-browser/lander-libp2p-config.js

@@ -10,19 +10,18 @@ import { circuitRelayTransport } from '@libp2p/circuit-relay-v2'
 const Libp2pBrowserOptions = {
   addresses: {
     listen: [
-      '/webrtc'
+      '/webrtc',
+      '/p2p-circuit'
     ]
   },
   transports: [
     webSockets({
       filter: all // connect to insecure sockets also (E.g. /ws/)
     }),
     webRTC(),
-    circuitRelayTransport({
-      discoverRelays: 1
-    })
+    circuitRelayTransport()
   ],
-  connectionEncryption: [
+  connectionEncrypters: [
     noise()
   ],
   streamMuxers: [

test/utils/test-config-browser/orbiter-libp2p-config.js

@@ -10,19 +10,18 @@ import { circuitRelayTransport } from '@libp2p/circuit-relay-v2'
 const Libp2pBrowserOptions = {
   addresses: {
     listen: [
-      '/webrtc'
+      '/webrtc',
+      '/p2p-circuit'
     ]
   },
   transports: [
     webSockets({
       filter: all // connect to insecure sockets also (E.g. /ws/)
     }),
     webRTC(),
-    circuitRelayTransport({
-      discoverRelays: 1
-    })
+    circuitRelayTransport()
   ],
-  connectionEncryption: [
+  connectionEncrypters: [
     noise()
   ],
   streamMuxers: [

test/utils/test-config/lander-libp2p-config.js

@@ -4,7 +4,7 @@ import { yamux } from '@chainsafe/libp2p-yamux'
 import { webSockets } from '@libp2p/websockets'
 import { gossipsub } from '@chainsafe/libp2p-gossipsub'
 import { tcp } from '@libp2p/tcp'
-import { mdns } from '@libp2p/mdns'
+// import { mdns } from '@libp2p/mdns'
 
 const Libp2pOptions = {
   addresses: {
@@ -17,7 +17,7 @@ const Libp2pOptions = {
     tcp(),
     webSockets()
   ],
-  connectionEncryption: [
+  connectionEncrypters: [
     noise()
   ],
   streamMuxers: [
@@ -27,13 +27,12 @@ const Libp2pOptions = {
     denyDialMultiaddr: () => false // allow dialling of private addresses.
   },
   peerDiscovery: [
-    mdns()
+    /* mdns() */
   ],
   services: {
     identify: identify(),
     pubsub: gossipsub({
-      emitSelf: true,
-      allowPublishToZeroPeers: true
+      emitSelf: true
     })
   }
 }