Upgrade orbitdb to 0.28.0 and its dependencies (libp2p@0.33.0, ipfs@0…

.github/workflows/testNodes.yml

@@ -25,4 +25,4 @@ jobs:
       - name: run test without tor
         run: DEBUG='waggle:dbSnap*,waggle:localTest*' ts-node src/nodeTest/testReplicate.ts --nodesCount 1 --timeThreshold 200 --entriesCount 1000 --no-useTor
       - name: run test with tor
-        run: DEBUG='waggle:dbSnap*,waggle:localTest*' ts-node src/nodeTest/testReplicate.ts --nodesCount 1 --timeThreshold 200 --entriesCount 20 --useTor
+        run: DEBUG='waggle:dbSnap*,waggle:localTest*' ts-node src/nodeTest/testReplicate.ts --nodesCount 1 --timeThreshold 500 --entriesCount 1000 --useTor

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## [Unreleased]
 
+* Upgrade: orbit-db@0.28.0, ipfs@0.60.2, libp2p@0.33.0 and their dependencies
+* Remove custom libp2p
+
+## [4.3.13]
+
 ### Added
 
 * Logs and retry logic in websocketOverTorTest

jestSetup.js

@@ -0,0 +1,5 @@
+const { Crypto } = require('@peculiar/webcrypto')
+
+console.log('Custom jest setup')
+const crypto = new Crypto();
+global.crypto = crypto;

package.json

@@ -11,7 +11,7 @@
     "version": "git add -A src",
     "postversion": "git push && git push --tags",
     "lint": "eslint --ext .jsx,.js,.ts,.tsx ./src/",
-    "test": "DEBUG=ipfs:*,waggle:* jest ./src/**/* --runInBand --verbose",
+    "test": "DEBUG=ipfs:*,waggle:* jest --runInBand --verbose",
     "test-ci": "jest ./src/**/* --runInBand --colors --ci --silent --verbose"
   },
   "repository": {
@@ -34,7 +34,10 @@
     ],
     "testRegex": "src/.*\\.test\\.(t|j)s$",
     "testEnvironment": "node",
-    "testTimeout": 40000
+    "testTimeout": 40000,
+    "setupFiles": [
+      "./jestSetup.js"
+    ]
   },
   "devDependencies": {
     "@types/crypto-js": "^4.0.2",
@@ -65,6 +68,7 @@
     "pkijs": "^2.1.96"
   },
   "dependencies": {
+    "@chainsafe/libp2p-noise": "^4.0.0",
     "@zbayapp/identity": "^3.3.3",
     "abortable-iterator": "^3.0.0",
     "class-validator": "^0.13.1",
@@ -79,30 +83,26 @@
     "find-free-port": "^2.0.0",
     "http-server": "^0.12.3",
     "https-proxy-agent": "^5.0.0",
-    "ipfs": "0.52",
+    "ipfs": "^0.60.2",
+    "ipfs-log": "5.4.1",
     "it-pipe": "^1.1.0",
     "joi": "^17.4.0",
-    "libp2p": "^0.29.3",
-    "libp2p-bootstrap": "^0.12.1",
-    "libp2p-gossipsub": "^0.6.1",
-    "libp2p-interfaces": "^0.7.1",
-    "libp2p-kad-dht": "^0.20.1",
-    "libp2p-mdns": "^0.15.0",
-    "libp2p-mplex": "^0.10.0",
-    "libp2p-noise": "^2.0.0",
-    "libp2p-pubsub-peer-discovery": "^3.0.0",
-    "libp2p-tcp": "^0.15.1",
-    "libp2p-websockets": "^0.14.0",
-    "multiaddr": "^8.0.0",
-    "orbit-db": "git://github.com/orbitdb/orbit-db.git#4f3fabb7bcf20f1314fd8e52fca556ce2852e46a",
-    "orbit-db-store": "~4.0.1",
-    "peer-id": "^0.14.0",
+    "libp2p": "^0.33.0",
+    "libp2p-bootstrap": "^0.13.0",
+    "libp2p-gossipsub": "^0.11.0",
+    "libp2p-kad-dht": "^0.24.2",
+    "libp2p-mplex": "^0.10.1",
+    "libp2p-websockets": "^0.16.2",
+    "multiaddr": "^10.0.1",
+    "orbit-db": "^0.28.0",
+    "orbit-db-io": "^1.0.2",
+    "orbit-db-store": "^4.3.2",
+    "peer-id": "^0.15.0",
     "socket.io": "3.0.5",
     "socks-proxy-agent": "^5.0.0",
     "streaming-iterables": "^5.0.2",
     "uint8arrays": "^1.1.0",
     "url": "^0.11.0",
-    "validator": "^13.6.0",
-    "wrtc": "^0.4.6"
+    "validator": "^13.6.0"
   }
 }

src/common/testUtils.ts

@@ -1,4 +1,5 @@
 import fp from 'find-free-port'
+import Libp2p from 'libp2p'
 import { HttpsProxyAgent } from 'https-proxy-agent'
 import { Response } from 'node-fetch'
 import path from 'path'
@@ -7,7 +8,6 @@ import tmp from 'tmp'
 import { ConnectionsManagerOptions, DataFromPems } from '../common/types'
 import { Config } from '../constants'
 import { ConnectionsManager } from '../libp2p/connectionsManager'
-import { Libp2pType } from '../libp2p/customLibp2p'
 import { createCertificatesTestHelper } from '../libp2p/tests/client-server'
 import WebsocketsOverTor from '../libp2p/websocketOverTor'
 import logger from '../logger'
@@ -65,7 +65,7 @@ export const createMinConnectionManager = (options: ConnectionsManagerOptions):
   })
 }
 
-export const createLibp2p = async (peerId: PeerId): Promise<Libp2pType> => {
+export const createLibp2p = async (peerId: PeerId): Promise<Libp2p> => {
   const [port] = await fp(1111)
   const virtPort = 443
   const pems = await createCertificatesTestHelper('address1.onion', 'address2.onion')

src/libp2p/connectionsManager.ts

@@ -1,22 +1,17 @@
-import * as os from 'os'
+import { NOISE } from '@chainsafe/libp2p-noise'
 import { Crypto } from '@peculiar/webcrypto'
-import { CryptoEngine, setEngine } from 'pkijs'
+import { Agent } from 'https'
 import { HttpsProxyAgent } from 'https-proxy-agent'
+import Libp2p, { Connection } from 'libp2p'
 import Bootstrap from 'libp2p-bootstrap'
 import Gossipsub from 'libp2p-gossipsub'
-import { Connection } from 'libp2p-gossipsub/src/interfaces'
 import KademliaDHT from 'libp2p-kad-dht'
 import Mplex from 'libp2p-mplex'
-import { NOISE } from 'libp2p-noise'
 import { Response } from 'node-fetch'
+import * as os from 'os'
 import PeerId from 'peer-id'
+import { CryptoEngine, setEngine } from 'pkijs'
 import { CertsData, ConnectionsManagerOptions } from '../common/types'
-import { ZBAY_DIR_PATH } from '../constants'
-import logger from '../logger'
-import IOProxy from '../socket/IOProxy'
-import initListeners from '../socket/listeners'
-import { Storage } from '../storage'
-import { Tor } from '../torManager'
 import {
   createLibp2pAddress,
   createLibp2pListenAddress,
@@ -25,7 +20,12 @@ import {
   torBinForPlatform,
   torDirForPlatform
 } from '../common/utils'
-import CustomLibp2p, { Libp2pType } from './customLibp2p'
+import { ZBAY_DIR_PATH } from '../constants'
+import logger from '../logger'
+import IOProxy from '../socket/IOProxy'
+import initListeners from '../socket/listeners'
+import { Storage } from '../storage'
+import { Tor } from '../torManager'
 import WebsocketsOverTor from './websocketOverTor'
 const log = logger('conn')
 
@@ -89,7 +89,7 @@ export class ConnectionsManager {
     )
   }
 
-  public readonly createAgent = () => {
+  public readonly createAgent = (): Agent => {
     if (this.socksProxyAgent || !this.agentPort || !this.agentHost) return
 
     log(`Creating https proxy agent: ${this.httpTunnelPort}`)
@@ -180,7 +180,7 @@ export class ConnectionsManager {
     bootstrapMultiaddrs: string[],
     certs: CertsData,
     targetPort: number
-  ): Promise<{ libp2p: Libp2pType, localAddress: string }> => {
+  ): Promise<{ libp2p: Libp2p, localAddress: string }> => {
     const localAddress = this.createLibp2pAddress(address, addressPort, peerId.toB58String())
     log(`Initializing libp2p for ${peerId.toB58String()}`)
     const libp2p = ConnectionsManager.createBootstrapNode({
@@ -258,7 +258,7 @@ export class ConnectionsManager {
     bootstrapMultiaddrsList,
     transportClass,
     targetPort
-  }): Libp2pType => {
+  }): Libp2p => {
     return ConnectionsManager.defaultLibp2pNode({
       peerId,
       listenAddrs,
@@ -284,8 +284,8 @@ export class ConnectionsManager {
     bootstrapMultiaddrsList,
     transportClass,
     targetPort
-  }): Libp2pType => {
-    return new CustomLibp2p({
+  }): Libp2p => {
+    return new Libp2p({
       peerId,
       addresses: {
         listen: listenAddrs

src/libp2p/websocketOverTor.test.ts

@@ -1,5 +1,5 @@
 import WebsocketsOverTor from './websocketOverTor'
-import Multiaddr from 'multiaddr'
+import { Multiaddr } from 'multiaddr'
 import { Tor } from '../torManager/index'
 import os from 'os'
 import fp from 'find-free-port'

src/libp2p/websocketOverTor.ts

@@ -1,4 +1,3 @@
-
 import withIs from 'class-is'
 import WebSockets from 'libp2p-websockets'
 import { AbortError } from 'abortable-iterator'
@@ -9,11 +8,12 @@ import createServer from 'it-ws/server'
 import toConnection from 'libp2p-websockets/src/socket-to-conn'
 import url from 'url'
 import os from 'os'
-import multiaddr from 'multiaddr'
+import { Multiaddr } from 'multiaddr'
 import debug from 'debug'
 import PeerId from 'peer-id'
 import https from 'https'
 import { dumpPEM } from './utils'
+import pDefer from 'p-defer'
 
 const log: any = debug('libp2p:websockets:listener:waggle')
 log.error = debug('libp2p:websockets:listener:waggle:error')
@@ -95,17 +95,33 @@ class WebsocketsOverTor extends WebSockets {
     }
   }
 
-  async _connect(ma: multiaddr, options: any = {}) {
+  async _connect(ma: Multiaddr, options: any = {}) {
     if (options.signal?.aborted) {
       throw new AbortError()
     }
     const cOpts = ma.toOptions()
     log('connect %s:%s', cOpts.host, cOpts.port)
+
+    const errorPromise = pDefer()
+    const errfn = (err) => {
+      const msg = `connection error: ${err.message as string}`
+      log.error(msg)
+
+      errorPromise.reject(err)
+    }
+
     const myUri = `${toUri(ma) as string}/?remoteAddress=${encodeURIComponent(this.localAddress)}`
 
     const rawSocket = connect(myUri, Object.assign({ binary: true }, options))
+
+    if (rawSocket.socket.on) {
+      rawSocket.socket.on('error', errfn)
+    } else {
+      rawSocket.socket.onerror = errfn
+    }
+
     if (!options.signal) {
-      await rawSocket.connected()
+      await Promise.race([rawSocket.connected(), errorPromise.promise])
 
       log(`${this.localAddress} connected %s`, ma)
       return rawSocket
@@ -117,7 +133,10 @@ class WebsocketsOverTor extends WebSockets {
     const abort = new Promise((resolve, reject) => {
       onAbort = () => {
         reject(new AbortError())
-        rawSocket.close()
+        // FIXME: https://github.com/libp2p/js-libp2p-websockets/issues/121
+        setTimeout(() => {
+          rawSocket.close()
+        })
       }
 
       // Already aborted?
@@ -126,7 +145,7 @@ class WebsocketsOverTor extends WebSockets {
     })
 
     try {
-      await Promise.race([abort, rawSocket.connected()])
+      await Promise.race([abort, errorPromise.promise, rawSocket.connected()])
     } finally {
       options.signal.removeEventListener('abort', onAbort)
     }
@@ -136,6 +155,7 @@ class WebsocketsOverTor extends WebSockets {
   }
 
   prepareListener = ({ handler, upgrader }) => {
+    log('prepareListener')
     const listener: any = new EventEmitter()
     const trackConn = (server, maConn) => {
       server.__connections.push(maConn)
@@ -161,7 +181,7 @@ class WebsocketsOverTor extends WebSockets {
       const query = url.parse(request.url, true).query
       log('server', query.remoteAddress)
       try {
-        maConn = toConnection(stream, { remoteAddr: multiaddr(query.remoteAddress.toString()) })
+        maConn = toConnection(stream, { remoteAddr: new Multiaddr(query.remoteAddress.toString()) })
         const peer = {
           id: PeerId.createFromB58String(query.remoteAddress.toString().split('/p2p/')[1]),
           multiaddrs: [maConn.remoteAddr]
@@ -197,7 +217,7 @@ class WebsocketsOverTor extends WebSockets {
       return server.close()
     }
 
-    listener.listen = (ma: multiaddr) => {
+    listener.listen = (ma: Multiaddr) => {
       listeningMultiaddr = ma
 
       const listenOptions = {
@@ -231,7 +251,7 @@ class WebsocketsOverTor extends WebSockets {
           Object.keys(netInterfaces).forEach(niKey => {
             netInterfaces[niKey].forEach(ni => {
               if (ni.family === 'IPv4') {
-                multiaddrs.push(multiaddr(m.toString().replace('0.0.0.0', ni.address)))
+                multiaddrs.push(new Multiaddr(m.toString().replace('0.0.0.0', ni.address)))
               }
             })
           })