feat: peerstore persistence

.aegir.js

@@ -45,7 +45,7 @@ const after = async () => {
 }
 
 module.exports = {
-  bundlesize: { maxSize: '179kB' },
+  bundlesize: { maxSize: '185kB' },
   hooks: {
     pre: before,
     post: after

doc/API.md

@@ -74,6 +74,7 @@ Creates an instance of Libp2p.
 | [options.dialer] | `object` | libp2p Dialer configuration
 | [options.metrics] | `object` | libp2p Metrics configuration
 | [options.peerId] | [`PeerId`][peer-id] | peerId instance (it will be created if not provided) |
+| [options.peerStore] | `object` | libp2p PeerStore configuration |
 
 For Libp2p configurations and modules details read the [Configuration Document](./CONFIGURATION.md).
 

doc/CONFIGURATION.md

@@ -505,6 +505,34 @@ const node = await Libp2p.create({
 })
 ```
 
+#### Configuring PeerStore
+
+PeerStore persistence is disabled in libp2p by default. You can enable and configure it as follows. Aside from enabled being `false` by default, it will need an implementation of a [datastore](https://github.com/ipfs/interface-datastore). Take into consideration that using the memory datastore will be ineffective for persistence.
+
+The threshold number represents the maximum number of "dirty peers" allowed in the PeerStore, i.e. peers that are not updated in the datastore. In this context, browser nodes should use a threshold of 1, since they might not "stop" properly in several scenarios and the PeerStore might end up with unflushed records when the window is closed.
+
+```js
+const Libp2p = require('libp2p')
+const TCP = require('libp2p-tcp')
+const MPLEX = require('libp2p-mplex')
+const SECIO = require('libp2p-secio')
+
+const LevelStore = require('datastore-level')
+
+const node = await Libp2p.create({
+  modules: {
+    transport: [TCP],
+    streamMuxer: [MPLEX],
+    connEncryption: [SECIO]
+  },
+  datastore: new LevelStore('path/to/store'),
+  peerStore: {
+    persistence: true, // Is persistence enabled (default: false)
+    threshold: 5 // Number of dirty peers allowed (default: 5)
+  }
+})
+```
+
 #### Customizing Transports
 
 Some Transports can be passed additional options when they are created. For example, `libp2p-webrtc-star` accepts an optional, custom `wrtc` implementation. In addition to libp2p passing itself and an `Upgrader` to handle connection upgrading, libp2p will also pass the options, if they are provided, from `config.transport`.

package.json

@@ -50,6 +50,7 @@
     "err-code": "^2.0.0",
     "events": "^3.1.0",
     "hashlru": "^2.3.0",
+    "interface-datastore": "^0.8.3",
     "ipfs-utils": "^2.2.0",
     "it-all": "^1.0.1",
     "it-buffer": "^0.1.2",

src/config.js

@@ -20,6 +20,10 @@ const DefaultConfig = {
   metrics: {
     enabled: false
   },
+  peerStore: {
+    persistence: false,
+    threshold: 5
+  },
   config: {
     dht: {
       enabled: false,

src/index.js

@@ -23,6 +23,7 @@ const Metrics = require('./metrics')
 const TransportManager = require('./transport-manager')
 const Upgrader = require('./upgrader')
 const PeerStore = require('./peer-store')
+const PersistentPeerStore = require('./peer-store/persistent')
 const Registrar = require('./registrar')
 const ping = require('./ping')
 const {
@@ -43,9 +44,15 @@ class Libp2p extends EventEmitter {
     // and add default values where appropriate
     this._options = validateConfig(_options)
 
-    this.datastore = this._options.datastore
     this.peerId = this._options.peerId
-    this.peerStore = new PeerStore()
+    this.datastore = this._options.datastore
+
+    this.peerStore = (this.datastore && this._options.peerStore.persistence)
+      ? new PersistentPeerStore({
+        datastore: this.datastore,
+        ...this._options.peerStore
+      })
+      : new PeerStore()
 
     // Addresses {listen, announce, noAnnounce}
     this.addresses = this._options.addresses
@@ -219,7 +226,8 @@ class Libp2p extends EventEmitter {
 
       this._discovery = new Map()
 
-      this.connectionManager.stop()
+      await this.peerStore.stop()
+      await this.connectionManager.stop()
 
       await Promise.all([
         this.pubsub && this.pubsub.stop(),
@@ -393,6 +401,9 @@ class Libp2p extends EventEmitter {
     // Listen on the provided transports
     await this.transportManager.listen()
 
+    // Start PeerStore
+    await this.peerStore.start()
+
     if (this._config.pubsub.enabled) {
       this.pubsub && this.pubsub.start()
     }

src/peer-store/README.md

@@ -83,7 +83,47 @@ Access to its underlying books:
 - `change:multiaadrs` - emitted when a known peer has a different set of multiaddrs.
 - `change:protocols` - emitted when a known peer supports a different set of protocols.
 
+## Data Persistence
+
+The data stored in the PeerStore can be persisted if configured appropriately. Keeping a record of the peers already discovered by the peer, as well as their known data aims to improve the efficiency of peers joining the network after being offline.
+
+The libp2p node will need to receive a [datastore](https://github.com/ipfs/interface-datastore), in order to persist this data across restarts. A [datastore](https://github.com/ipfs/interface-datastore) stores its data in a key-value fashion. As a result, we need coherent keys so that we do not overwrite data.
+
+The PeerStore should not continuously update the datastore whenever data is changed. Instead, it should only store new data after reaching a certain threshold of "dirty" peers, as well as when the node is stopped, in order to batch writes to the datastore.
+
+The peer id will be appended to the datastore key for each data namespace. The namespaces were defined as follows:
+
+**AddressBook**
+
+All the known peer addresses are stored with a key pattern as follows:
+
+`/peers/addrs/<b32 peer id no padding>`
+
+**ProtoBook**
+
+All the known peer protocols are stored with a key pattern as follows:
+
+`/peers/protos/<b32 peer id no padding>`
+
+**KeyBook**
+
+_NOT_YET_IMPLEMENTED_
+
+All public keys are stored under the following pattern:
+
+` /peers/keys/<b32 peer id no padding>`
+
+**MetadataBook**
+
+_NOT_YET_IMPLEMENTED_
+
+Metadata is stored under the following key pattern:
+
+`/peers/metadata/<b32 peer id no padding>/<key>`
+
 ## Future Considerations
 
 - If multiaddr TTLs are added, the PeerStore may schedule jobs to delete all addresses that exceed the TTL to prevent AddressBook bloating
 - Further API methods will probably need to be added in the context of multiaddr validity and confidence.
+- When improving libp2p configuration for specific runtimes, we should take into account the PeerStore recommended datastore.
+- When improving libp2p configuration, we should think about a possible way of allowing the configuration of Bootstrap to be influenced by the persisted peers, as a way to decrease the load on Bootstrap nodes.

src/peer-store/address-book.js

@@ -11,7 +11,7 @@ const PeerId = require('peer-id')
 const Book = require('./book')
 
 const {
-  ERR_INVALID_PARAMETERS
+  codes: { ERR_INVALID_PARAMETERS }
 } = require('../errors')
 
 /**
@@ -35,7 +35,12 @@ class AddressBook extends Book {
      * "peer" - emitted when a peer is discovered by the node.
      * "change:multiaddrs" - emitted when the known multiaddrs of a peer change.
      */
-    super(peerStore, 'change:multiaddrs', 'multiaddrs')
+    super({
+      peerStore,
+      eventName: 'change:multiaddrs',
+      eventProperty: 'multiaddrs',
+      eventTransformer: (data) => data.map((address) => address.multiaddr)
+    })
 
     /**
      * Map known peers to their known Addresses.
@@ -78,20 +83,14 @@ class AddressBook extends Book {
       }
     }
 
-    this.data.set(id, addresses)
-    this._setPeerId(peerId)
+    this._setData(peerId, addresses)
     log(`stored provided multiaddrs for ${id}`)
 
     // Notify the existance of a new peer
     if (!rec) {
       this._ps.emit('peer', peerId)
     }
 
-    this._ps.emit('change:multiaddrs', {
-      peerId,
-      multiaddrs: addresses.map((mi) => mi.multiaddr)
-    })
-
     return this
   }
 
@@ -127,16 +126,10 @@ class AddressBook extends Book {
       return this
     }
 
-    this._setPeerId(peerId)
-    this.data.set(id, addresses)
+    this._setData(peerId, addresses)
 
     log(`added provided multiaddrs for ${id}`)
 
-    this._ps.emit('change:multiaddrs', {
-      peerId,
-      multiaddrs: addresses.map((mi) => mi.multiaddr)
-    })
-
     // Notify the existance of a new peer
     if (!rec) {
       this._ps.emit('peer', peerId)
@@ -147,6 +140,7 @@ class AddressBook extends Book {
 
   /**
    * Transforms received multiaddrs into Address.
+   * @private
    * @param {Array<Multiaddr>} multiaddrs
    * @returns {Array<Address>}
    */

src/peer-store/book.js

@@ -4,17 +4,28 @@ const errcode = require('err-code')
 const PeerId = require('peer-id')
 
 const {
-  ERR_INVALID_PARAMETERS
+  codes: { ERR_INVALID_PARAMETERS }
 } = require('../errors')
 
+const passthrough = data => data
+
 /**
  * The Book is the skeleton for the PeerStore books.
  */
 class Book {
-  constructor (peerStore, eventName, eventProperty) {
+  /**
+   * @constructor
+   * @param {Object} properties
+   * @param {PeerStore} properties.peerStore PeerStore instance.
+   * @param {string} properties.eventName Name of the event to emit by the PeerStore.
+   * @param {string} properties.eventProperty Name of the property to emit by the PeerStore.
+   * @param {function} [properties.eventTransformer] Transformer function of the provided data for being emitted.
+   */
+  constructor ({ peerStore, eventName, eventProperty, eventTransformer = passthrough }) {
     this._ps = peerStore
     this.eventName = eventName
     this.eventProperty = eventProperty
+    this.eventTransformer = eventTransformer
 
     /**
      * Map known peers to their data.
@@ -32,6 +43,29 @@ class Book {
     throw errcode(new Error('set must be implemented by the subclass'), 'ERR_NOT_IMPLEMENTED')
   }
 
+  /**
+   * Set data into the datastructure, persistence and emit it using the provided transformers.
+   * @private
+   * @param {PeerId} peerId peerId of the data to store
+   * @param {Array<*>} data data to store.
+   * @param {Object} [options] storing options.
+   * @param {boolean} [options.emit = true] emit the provided data.
+   * @return {void}
+   */
+  _setData (peerId, data, { emit = true } = {}) {
+    const b58key = peerId.toB58String()
+
+    // Store data in memory
+    this.data.set(b58key, data)
+    this._setPeerId(peerId)
+
+    // Emit event
+    emit && this._ps.emit(this.eventName, {
+      peerId,
+      [this.eventProperty]: this.eventTransformer(data)
+    })
+  }
+
   /**
    * Add known data of a provided peer.
    * @param {PeerId} peerId
@@ -78,6 +112,11 @@ class Book {
     return true
   }
 
+  /**
+   * Set PeerId into peerStore datastructure.
+   * @private
+   * @param {PeerId} peerId
+   */
   _setPeerId (peerId) {
     if (!this._ps.peerIds.get(peerId)) {
       this._ps.peerIds.set(peerId.toB58String(), peerId)

src/peer-store/index.js

@@ -30,11 +30,14 @@ class PeerStore extends EventEmitter {
    * @property {Array<string>} protocols peer's supported protocols.
    */
 
+  /**
+   * @constructor
+   */
   constructor () {
     super()
 
     /**
-     * AddressBook containing a map of peerIdStr to Address
+     * AddressBook containing a map of peerIdStr to Address.
      */
     this.addressBook = new AddressBook(this)
 
@@ -51,6 +54,16 @@ class PeerStore extends EventEmitter {
     this.peerIds = new Map()
   }
 
+  /**
+   * Start the PeerStore.
+   */
+  start () {}
+
+  /**
+   * Stop the PeerStore.
+   */
+  stop () {}
+
   /**
    * Get all the stored information of every peer.
    * @returns {Map<string, Peer>}

src/peer-store/persistent/consts.js

@@ -0,0 +1,9 @@
+'use strict'
+
+module.exports.NAMESPACE_COMMON = '/peers/'
+
+// /peers/protos/<b32 peer id no padding>
+module.exports.NAMESPACE_ADDRESS = '/peers/addrs/'
+
+// /peers/addrs/<b32 peer id no padding>
+module.exports.NAMESPACE_PROTOCOL = '/peers/protos/'