Use newer js-ipfs proxy libraries (#288)

* Stop pinning

* Upgrade js-ipfs proxy libraries

* It's alive!

* v2 ipfs worker

* Update changelog

* Fix most of the tests

* Use dag.import

* Bump version to 0.29

Co-authored-by: Philipp Krüger <philipp.krueger1@gmail.com>

CHANGELOG.md

@@ -1,8 +1,10 @@
 # Changelog
 
-### v0.28.2
+### v0.29.0
 
-- Update API endpoints to v2 and add setup parameter to specify API version
+- Upgrade js-ipfs libraries to the versions corresponding to the 0.58 release.
+- Update API endpoints to v2 and add setup parameter to specify API version.
+- No longer uses `ipfs-message-port-client` and `ipfs-message-port-protocol` forks which sometimes caused weird dependency conflicts.
 
 
 ### v0.28.1

package.json

@@ -1,6 +1,6 @@
 {
   "name": "webnative",
-  "version": "0.28.2",
+  "version": "0.29.0",
   "description": "Fission Webnative SDK",
   "keywords": [
     "WebCrypto",
@@ -72,14 +72,15 @@
     "@types/throttle-debounce": "^2.1.0",
     "@typescript-eslint/eslint-plugin": "^4.29.3",
     "@typescript-eslint/parser": "^4.29.3",
+    "blockstore-core": "^1.0.2",
+    "datastore-core": "^6.0.7",
     "esbuild": "^0.12.8",
     "eslint": "^7.27.0",
     "expect": "^27.0.6",
     "fast-check": "^2.14.0",
-    "interface-datastore": "^4.0.1",
-    "ipfs-core": "0.8.0",
-    "ipfs-core-types": "0.5.2",
-    "ipfs-repo": "^9.1.6",
+    "ipfs-core": "0.11.2-rc.2",
+    "ipfs-core-types": "0.8.2-rc.2",
+    "ipfs-repo": "^13.0.5",
     "mocha": "^9.0.2",
     "multihashing-async": "^2.1.2",
     "puppeteer": "^10.1.0",
@@ -91,23 +92,24 @@
     "yarn": "^1.22.4"
   },
   "dependencies": {
+    "@ipld/dag-pb": "^2.1.11",
     "cborg": "^1.3.4",
     "cids": "^1.1.6",
     "fission-bloom-filters": "1.7.1",
-    "ipfs-message-port-client": "https://ipfs.runfission.com/ipfs/bafybeigx6q4aezve7my76s5vvfuiinbxtepapqvmjf2jbgrozrut6cjape/p/ipfs-message-port-client.tar.gz",
-    "ipfs-message-port-protocol": "https://ipfs.runfission.com/ipfs/bafybeigx6q4aezve7my76s5vvfuiinbxtepapqvmjf2jbgrozrut6cjape/p/ipfs-message-port-protocol.tar.gz",
+    "ipfs-message-port-client": "^0.9.2-rc.2",
+    "ipfs-message-port-protocol": "^0.10.2-rc.2",
     "ipld-dag-pb": "^0.22.2",
     "keystore-idb": "^0.15.1",
     "localforage": "^1.9.0",
-    "multiformats": "^9.0.0",
+    "multiformats": "^9.4.8",
     "throttle-debounce": "^3.0.1",
     "tweetnacl": "^0.14.5",
     "uint8arrays": "^2.1.7"
   },
-  "resolutions": {
-    "**/ipfs-message-port-protocol": "https://ipfs.runfission.com/ipfs/bafybeigx6q4aezve7my76s5vvfuiinbxtepapqvmjf2jbgrozrut6cjape/p/ipfs-message-port-protocol.tar.gz"
-  },
   "tsd": {
     "directory": "lib"
+  },
+  "resolutions": {
+    "multiformats": "^9.4.8"
   }
 }

src/common/version.ts

@@ -1 +1 @@
-export const VERSION = "0.28.2"
+export const VERSION = "0.29.0"

src/fs/protocol/private/mmpt.node.test.ts

@@ -31,7 +31,7 @@ async function generateExampleEntries(ipfs: IPFS, amount: number): Promise<{ nam
     const cid = await ipfs.object.put({ Data: encode(hash), Links: [] })
     entries.push({
       name: hash,
-      cid: cid.toBaseEncodedString(),
+      cid: cid.toString(),
     })
   }
 

src/ipfs/basic.ts

@@ -1,5 +1,6 @@
-import CIDObj from "cids"
+import { CID as CIDObj } from "multiformats/cid"
 import dagPb, { DAGLink, DAGNode } from "ipld-dag-pb"
+import * as dagPB from "@ipld/dag-pb"
 import type { IPFSEntry } from "ipfs-core-types/src/root"
 import type { ImportCandidate } from "ipfs-core-types/src/utils"
 
@@ -26,7 +27,7 @@ export const catRaw = async (cid: CID): Promise<Uint8Array[]> => {
   const ipfs = await getIpfs()
   const chunks = []
   await attemptPin(cid)
-  for await (const chunk of ipfs.cat(cid)) {
+  for await (const chunk of ipfs.cat(CIDObj.parse(cid))) {
     chunks.push(chunk)
   }
   return chunks
@@ -45,7 +46,7 @@ export const cat = async (cid: CID): Promise<string> => {
 export const ls = async (cid: CID): Promise<IPFSEntry[]> => {
   const ipfs = await getIpfs()
   const links = []
-  for await (const link of ipfs.ls(cid)) {
+  for await (const link of ipfs.ls(CIDObj.parse(cid))) {
     links.push(link)
   }
   return links
@@ -54,16 +55,24 @@ export const ls = async (cid: CID): Promise<IPFSEntry[]> => {
 export const dagGet = async (cid: CID): Promise<DAGNode> => {
   const ipfs = await getIpfs()
   await attemptPin(cid)
-  const raw = await ipfs.dag.get(new CIDObj(cid))
+  const raw = await ipfs.dag.get(CIDObj.parse(cid))
   const node = util.rawToDAGNode(raw)
   return node
 }
 
 export const dagPut = async (node: DAGNode): Promise<AddResult> => {
   const ipfs = await getIpfs()
+  const newNode = dagPB.createNode(
+    node.Data,
+    node.Links.map(link => dagPB.createLink(
+      link.Name,
+      link.Tsize,
+      CIDObj.decode(link.Hash.bytes)
+    ))
+  )
   // using this format because Gateway doesn't like `dag-cbor` nodes.
   // I think this is because UnixFS requires `dag-pb` & the gateway requires UnixFS for directory traversal
-  const cidObj = await ipfs.dag.put(node, { format: "dag-pb", hashAlg: "sha2-256" })
+  const cidObj = await ipfs.dag.put(newNode, { format: "dag-pb", hashAlg: "sha2-256" })
   const cid = cidObj.toV1().toString()
   await attemptPin(cid)
   const nodeSize = await size(cid)

src/ipfs/config.ts

@@ -1,5 +1,6 @@
-import IpfsMessagePortClient from "ipfs-message-port-client"
+import { IPFSClient } from "ipfs-message-port-client"
 import type { IPFS } from "ipfs-core"
+
 import { setup } from "../setup/internal.js"
 
 
@@ -13,7 +14,7 @@ export const set = (userIpfs: unknown): void => {
 export const get = async (): Promise<IPFS> => {
   if (!ipfs) {
     const port = await iframe()
-    ipfs = IpfsMessagePortClient.from(port) as unknown as IPFS
+    ipfs = IPFSClient.from(port) as unknown as IPFS
   }
   return ipfs
 }
@@ -36,6 +37,6 @@ export function iframe(): Promise<MessagePort> {
         : reject(new Error("Don't have access to iframe.contentWindow"))
     }
 
-    iframe.src = `${setup.endpoints.lobby}/ipfs.html`
+    iframe.src = `${setup.endpoints.lobby}/ipfs/v2.html`
   })
 }

src/ipfs/util.ts

@@ -1,6 +1,7 @@
+import { CID } from "multiformats/cid"
 import dagPb, { DAGLink, DAGNode } from "ipld-dag-pb"
 import type { GetResult } from "ipfs-core-types/src/dag"
-import { CID } from "ipfs-message-port-client/src/block"
+import OldCID from "cids"
 
 type RawDAGLink = {
   Name: string
@@ -9,7 +10,9 @@ type RawDAGLink = {
 }
 
 const rawToDAGLink = (raw: RawDAGLink): DAGLink => {
-  return new dagPb.DAGLink(raw.Name, raw.Tsize, raw.Hash)
+  const h = raw.Hash
+  const c = new OldCID(h.version, h.code, h.multihash.bytes)
+  return new dagPb.DAGLink(raw.Name, raw.Tsize, c)
 }
 
 export const rawToDAGNode = (raw: GetResult): DAGNode => {

src/setup/internal.ts

@@ -19,5 +19,5 @@ export const setup = {
     user: "fission.name"
   },
 
-  shouldPin: true,
+  shouldPin: false,
 }

tests/helpers/in-memory-ipfs.ts

@@ -1,8 +1,23 @@
-import Ipfs, { IPFS } from "ipfs-core"
-import Repo from "ipfs-repo"
-import { MemoryDatastore } from "interface-datastore"
+import * as fs from "fs"
+import * as dagPB from "@ipld/dag-pb"
+import * as Ipfs from "ipfs-core"
+
+import { createRepo } from "ipfs-repo"
+import { BlockCodec } from "multiformats/codecs/interface"
+import { MemoryDatastore } from "datastore-core/memory"
+import { MemoryBlockstore } from "blockstore-core/memory"
+
+import tempDir from "ipfs-utils/src/temp-dir.js"
+import type { IPFS } from "ipfs-core"
+
 
 export async function createInMemoryIPFS(): Promise<IPFS> {
+  const dir = tempDir()
+  fs.mkdirSync(dir)
+
+  const memoryDs = new MemoryDatastore()
+  const memoryBs = new MemoryBlockstore()
+
   return await Ipfs.create({
     offline: true,
     silent: true,
@@ -14,19 +29,28 @@ export async function createInMemoryIPFS(): Promise<IPFS> {
         Swarm: []
       },
     },
-    repo: new Repo("inmem", {
-      lock: {
-        lock: async () => ({ close: async () => { return } }),
-        locked: async () => false
-      },
-      autoMigrate: false,
-      storageBackends: {
-        root: MemoryDatastore,
-        blocks: MemoryDatastore,
-        keys: MemoryDatastore,
-        datastore: MemoryDatastore,
-        pins: MemoryDatastore,
-      },
-    })
+    repo: createRepo(
+      dir,
+      codeOrName => {
+        const lookup: Record<string, BlockCodec<number, unknown>> = {
+          [dagPB.code]: dagPB,
+          [dagPB.name]: dagPB
+        }
+
+        return Promise.resolve(lookup[codeOrName])
+      }, {
+        root: memoryDs,
+        blocks: memoryBs,
+        keys: memoryDs,
+        datastore: memoryDs,
+        pins: memoryDs
+      }, {
+        repoLock: {
+          lock: async () => ({ close: async () => { return } }),
+          locked: async () => false
+        },
+        autoMigrate: false,
+      }
+    )
   })
 }

tests/helpers/loadCAR.ts

@@ -1,26 +1,17 @@
 import * as fs from "fs"
+import all from "it-all"
 import { IPFS } from "ipfs-core"
 import { CID } from "multiformats"
-import { CarBlockIterator } from "@ipld/car"
 
 /**
  * @returns the roots defined in the CAR file
  */
-export async function loadCAR(filepath: string, ipfs: IPFS): Promise<{ roots: CID[]; cids: CID[] }> {
+export async function loadCAR(filepath: string, ipfs: IPFS): Promise<{ roots: CID[] }> {
   const inStream = fs.createReadStream(filepath)
   try {
-    const cids: CID[] = []
-    const blockIterator = await CarBlockIterator.fromIterable(inStream)
-    for await (const block of blockIterator) {
-      cids.push(block.cid)
-      await ipfs.block.put(block.bytes, { cid: block.cid })
-    }
-    return {
-      roots: await blockIterator.getRoots(),
-      cids,
-    }
+    const roots = await all(ipfs.dag.import(inStream))
+    return { roots: roots.map(root => root.root.cid) }
   } finally {
     inStream.close()
   }
 }
-