feat: load ipfs-webui from IPFS

Before, webui was bundled with extension itself. Unfortunately this
caused issues with reviews at addon (webui build was not reproducible).

This change replaces bundled webui with one loaded from custom gateway.
To make it work we modify HTTP headers to make cross-origin requests
from 'blessed' webuiRootUrl work without changing default go-ipfs configuration.

To improve UX, content script sets `ipfsApi` in webui's `localStorage`
to the same endpoint as one defined in Companion (unless it was already
customized by the user).

add-on/src/lib/ipfs-companion.js

@@ -2,6 +2,7 @@
 /* eslint-env browser, webextensions */
 
 const browser = require('webextension-polyfill')
+const toMultiaddr = require('uri-to-multiaddr')
 const { optionDefaults, storeMissingOptions, migrateOptions } = require('./options')
 const { initState, offlinePeerCount } = require('./state')
 const { createIpfsPathValidator, pathAtHttpGateway } = require('./ipfs-path')
@@ -92,13 +93,18 @@ module.exports = async function init () {
   }
 
   function registerListeners () {
-    browser.webRequest.onBeforeSendHeaders.addListener(onBeforeSendHeaders, { urls: ['<all_urls>'] }, ['blocking', 'requestHeaders'])
+    let onBeforeSendInfoSpec = ['blocking', 'requestHeaders']
+    if (!runtime.isFirefox) {
+      // Chrome 72+  requires 'extraHeaders' for access to Referer header (used in cors whitelisting of webui)
+      onBeforeSendInfoSpec.push('extraHeaders')
+    }
+    browser.webRequest.onBeforeSendHeaders.addListener(onBeforeSendHeaders, { urls: ['<all_urls>'] }, onBeforeSendInfoSpec)
     browser.webRequest.onBeforeRequest.addListener(onBeforeRequest, { urls: ['<all_urls>'] }, ['blocking'])
     browser.webRequest.onHeadersReceived.addListener(onHeadersReceived, { urls: ['<all_urls>'] }, ['blocking', 'responseHeaders'])
     browser.webRequest.onErrorOccurred.addListener(onErrorOccurred, { urls: ['<all_urls>'] })
     browser.storage.onChanged.addListener(onStorageChange)
     browser.webNavigation.onCommitted.addListener(onNavigationCommitted)
-    browser.tabs.onUpdated.addListener(onUpdatedTab)
+    browser.webNavigation.onDOMContentLoaded.addListener(onDOMContentLoaded)
     browser.tabs.onActivated.addListener(onActivatedTab)
     if (browser.windows) {
       browser.windows.onFocusChanged.addListener(onWindowFocusChanged)
@@ -213,6 +219,7 @@ module.exports = async function init () {
       peerCount: state.peerCount,
       gwURLString: state.gwURLString,
       pubGwURLString: state.pubGwURLString,
+      webuiRootUrl: state.webuiRootUrl,
       currentTab: await browser.tabs.query({ active: true, currentWindow: true }).then(tabs => tabs[0])
     }
     try {
@@ -378,45 +385,57 @@ module.exports = async function init () {
     }
   }
 
-  async function onUpdatedTab (tabId, changeInfo, tab) {
+  async function onDOMContentLoaded (details) {
     if (!state.active) return // skip content script injection when off
-    if (changeInfo.status && changeInfo.status === 'complete' && tab.url && tab.url.startsWith('http')) {
-      if (state.linkify) {
-        console.info(`[ipfs-companion] Running linkfyDOM for ${tab.url}`)
-        try {
-          await browser.tabs.executeScript(tabId, {
-            file: '/dist/bundles/linkifyContentScript.bundle.js',
-            matchAboutBlank: false,
-            allFrames: true,
-            runAt: 'document_idle'
-          })
-        } catch (error) {
-          console.error(`Unable to linkify DOM at '${tab.url}' due to`, error)
-        }
+    if (!details.url.startsWith('http')) return // skip special pages
+    // console.info(`[ipfs-companion] onDOMContentLoaded`, details)
+    if (state.linkify) {
+      console.info(`[ipfs-companion] Running linkfy experiment for ${details.url}`)
+      try {
+        await browser.tabs.executeScript(details.tabId, {
+          file: '/dist/bundles/linkifyContentScript.bundle.js',
+          matchAboutBlank: false,
+          allFrames: true,
+          runAt: 'document_idle'
+        })
+      } catch (error) {
+        console.error(`Unable to linkify DOM at '${details.url}' due to`, error)
       }
-      if (state.catchUnhandledProtocols) {
-        // console.log(`[ipfs-companion] Normalizing links with unhandled protocols at ${tab.url}`)
-        // See: https://github.com/ipfs/ipfs-companion/issues/286
-        try {
-          // pass the URL of user-preffered public gateway
-          await browser.tabs.executeScript(tabId, {
-            code: `window.ipfsCompanionPubGwURL = '${state.pubGwURLString}'`,
-            matchAboutBlank: false,
-            allFrames: true,
-            runAt: 'document_start'
-          })
-          // inject script that normalizes `href` and `src` containing unhandled protocols
-          await browser.tabs.executeScript(tabId, {
-            file: '/dist/bundles/normalizeLinksContentScript.bundle.js',
-            matchAboutBlank: false,
-            allFrames: true,
-            runAt: 'document_end'
-          })
-        } catch (error) {
-          console.error(`Unable to normalize links at '${tab.url}' due to`, error)
-        }
+    }
+    if (state.catchUnhandledProtocols) {
+      // console.log(`[ipfs-companion] Normalizing links with unhandled protocols at ${tab.url}`)
+      // See: https://github.com/ipfs/ipfs-companion/issues/286
+      try {
+        // pass the URL of user-preffered public gateway
+        await browser.tabs.executeScript(details.tabId, {
+          code: `window.ipfsCompanionPubGwURL = '${state.pubGwURLString}'`,
+          matchAboutBlank: false,
+          allFrames: true,
+          runAt: 'document_start'
+        })
+        // inject script that normalizes `href` and `src` containing unhandled protocols
+        await browser.tabs.executeScript(details.tabId, {
+          file: '/dist/bundles/normalizeLinksContentScript.bundle.js',
+          matchAboutBlank: false,
+          allFrames: true,
+          runAt: 'document_end'
+        })
+      } catch (error) {
+        console.error(`Unable to normalize links at '${details.url}' due to`, error)
       }
     }
+    if (details.url.startsWith(state.webuiRootUrl)) {
+      // Ensure API backend points at one from IPFS Companion
+      const apiMultiaddr = toMultiaddr(state.apiURLString)
+      await browser.tabs.executeScript(details.tabId, {
+        runAt: 'document_start',
+        code: `if (!localStorage.getItem('ipfsApi')) {
+          console.log('[ipfs-companion] Setting API to ${apiMultiaddr}');
+          localStorage.setItem('ipfsApi', '${apiMultiaddr}');
+          window.location.reload();
+        }`
+      })
+    }
   }
 
   // API STATUS UPDATES
@@ -597,6 +616,7 @@ module.exports = async function init () {
         case 'customGatewayUrl':
           state.gwURL = new URL(change.newValue)
           state.gwURLString = state.gwURL.toString()
+          state.webuiRootUrl = `${state.gwURLString}ipfs/${state.webuiCid}/`
           break
         case 'publicGatewayUrl':
           state.pubGwURL = new URL(change.newValue)

add-on/src/lib/ipfs-request.js

@@ -25,10 +25,33 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
   const runtimeRoot = browser.runtime.getURL('/')
   const webExtensionOrigin = runtimeRoot ? new URL(runtimeRoot).origin : 'null'
 
-  // Ignored requests are identified once and cached across all browser.webRequest hooks
-  const ignoredRequests = new LRU({ max: 128, maxAge: 1000 * 30 })
+  // Various types of requests are identified once and cached across all browser.webRequest hooks
+  const requestCacheCfg = { max: 128, maxAge: 1000 * 30 }
+  const ignoredRequests = new LRU(requestCacheCfg)
   const ignore = (id) => ignoredRequests.set(id, true)
   const isIgnored = (id) => ignoredRequests.get(id) !== undefined
+
+  const acrhHeaders = new LRU(requestCacheCfg) // webui cors fix in Chrome
+  const originUrls = new LRU(requestCacheCfg) // request.originUrl workaround for Chrome
+  const originUrl = (request) => {
+    // Firefox and Chrome provide relevant value in different fields:
+    // (Firefox) request object includes full URL of origin document, return as-is
+    if (request.originUrl) return request.originUrl
+    // (Chrome) is lacking: `request.initiator` is just the origin (protocol+hostname+port)
+    // To reconstruct originUrl we read full URL from Referer header in onBeforeSendHeaders
+    // and cache it for short time
+    // TODO: when request.originUrl is available in Chrome the `originUrls` cache can be removed
+    let cachedUrl = originUrls.get(request.requestId)
+    if (cachedUrl) return cachedUrl
+    if (request.requestHeaders) {
+      const referer = request.requestHeaders.find(h => h.name === 'Referer')
+      if (referer) {
+        originUrls.set(request.requestId, referer.value)
+        return referer.value
+      }
+    }
+  }
+
   const preNormalizationSkip = (state, request) => {
     // skip requests to the custom gateway or API (otherwise we have too much recursion)
     if (request.url.startsWith(state.gwURLString) || request.url.startsWith(state.apiURLString)) {
@@ -44,6 +67,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     }
     return isIgnored(request.requestId)
   }
+
   const postNormalizationSkip = (state, request) => {
     // skip requests to the public gateway if embedded node is running (otherwise we have too much recursion)
     if (state.ipfsNodeType === 'embedded' && request.url.startsWith(state.pubGwURLString)) {
@@ -115,12 +139,40 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
         return
       }
 
+      // Special handling of requests made to API
       if (request.url.startsWith(state.apiURLString)) {
+        // Requests made by 'blessed' Web UI
+        // --------------------------------------------
+        // Goal: Web UI works without setting CORS at go-ipfs
+        // (Without this snippet go-ipfs will return HTTP 403 due to additional origin check on the backend)
+        const origin = originUrl(request)
+        if (origin && origin.startsWith(state.webuiRootUrl)) {
+          // console.log('onBeforeSendHeaders', request)
+          // console.log('onBeforeSendHeaders.origin', origin)
+          // Swap Origin to pass server-side check
+          // (go-ipfs returns HTTP 403 on origin mismatch if there are no CORS headers)
+          const swapOrigin = (at) => {
+            request.requestHeaders[at].value = request.requestHeaders[at].value.replace(state.gwURL.origin, state.apiURL.origin)
+          }
+          let foundAt = request.requestHeaders.findIndex(h => h.name === 'Origin')
+          if (foundAt > -1) swapOrigin(foundAt)
+          foundAt = request.requestHeaders.findIndex(h => h.name === 'Referer')
+          if (foundAt > -1) swapOrigin(foundAt)
+
+          // Save access-control-request-headers from preflight
+          foundAt = request.requestHeaders.findIndex(h => h.name && h.name.toLowerCase() === 'access-control-request-headers')
+          if (foundAt > -1) {
+            acrhHeaders.set(request.requestId, request.requestHeaders[foundAt].value)
+            // console.log('onBeforeSendHeaders FOUND access-control-request-headers', acrhHeaders.get(request.requestId))
+          }
+          // console.log('onBeforeSendHeaders fixed headers', request.requestHeaders)
+        }
+
         // '403 - Forbidden' fix for Chrome and Firefox
         // --------------------------------------------
-        // We remove Origin header from requests made to API URL
+        // We remove Origin header from requests made to API URL and WebUI
         // by js-ipfs-http-client running in WebExtension context to remove need
-        // for manual whitelisting Access-Control-Allow-Origin at go-ipfs
+        // for manual CORS whitelisting via Access-Control-Allow-Origin at go-ipfs
         // More info:
         // Firefox: https://github.com/ipfs-shipyard/ipfs-companion/issues/622
         // Chromium 71: https://github.com/ipfs-shipyard/ipfs-companion/pull/616
@@ -142,13 +194,10 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
           }
           return false
         }
-        for (let i = 0; i < request.requestHeaders.length; i++) {
-          let header = request.requestHeaders[i]
-          if (header.name === 'Origin' && isWebExtensionOrigin(header.value)) {
-            request.requestHeaders.splice(i, 1)
-            break
-          }
-        }
+
+        // Remove Origin header matching webExtensionOrigin
+        const foundAt = request.requestHeaders.findIndex(h => h.name === 'Origin' && isWebExtensionOrigin(h.value))
+        if (foundAt > -1) request.requestHeaders.splice(foundAt, 1)
 
         // Fix "http: invalid Read on closed Body"
         // ----------------------------------
@@ -200,8 +249,48 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     onHeadersReceived (request) {
       const state = getState()
 
-      // Skip if IPFS integrations are inactive or request is marked as ignored
-      if (!state.active || isIgnored(request.requestId)) {
+      // Skip if IPFS integrations are inactive
+      if (!state.active) {
+        return
+      }
+
+      // Special handling of requests made to API
+      if (request.url.startsWith(state.apiURLString)) {
+        // Special handling of requests made by 'blessed' Web UI from local Gateway
+        // Goal: Web UI works without setting CORS at go-ipfs
+        // (This includes 'ignored' requests: CORS needs to be fixed even if no redirect is done)
+        const origin = originUrl(request)
+        if (origin && origin.startsWith(state.webuiRootUrl) && request.responseHeaders) {
+          // console.log('onHeadersReceived', request)
+          const acaOriginHeader = { name: 'Access-Control-Allow-Origin', value: state.gwURL.origin }
+          const foundAt = findHeaderIndex(acaOriginHeader.name, request.responseHeaders)
+          if (foundAt > -1) {
+            request.responseHeaders[foundAt].value = acaOriginHeader.value
+          } else {
+            request.responseHeaders.push(acaOriginHeader)
+          }
+
+          // Restore access-control-request-headers from preflight
+          const acrhValue = acrhHeaders.get(request.requestId)
+          if (acrhValue) {
+            const acahHeader = { name: 'Access-Control-Allow-Headers', value: acrhValue }
+            const foundAt = findHeaderIndex(acahHeader.name, request.responseHeaders)
+            if (foundAt > -1) {
+              request.responseHeaders[foundAt].value = acahHeader.value
+            } else {
+              request.responseHeaders.push(acahHeader)
+            }
+            acrhHeaders.del(request.requestId)
+            // console.log('onHeadersReceived SET  Access-Control-Allow-Headers', header)
+          }
+
+          // console.log('onHeadersReceived fixed headers', request.responseHeaders)
+          return { responseHeaders: request.responseHeaders }
+        }
+      }
+
+      // Skip if request is marked as ignored
+      if (isIgnored(request.requestId)) {
         return
       }
 
@@ -419,3 +508,7 @@ function normalizedUnhandledIpfsProtocol (request, pubGwUrl) {
     return { redirectUrl: pathAtHttpGateway(path, pubGwUrl) }
   }
 }
+
+function findHeaderIndex (name, headers) {
+  return headers.findIndex(x => x.name && x.name.toLowerCase() === name.toLowerCase())
+}

add-on/src/lib/state.js

@@ -22,6 +22,10 @@ function initState (options) {
   state.gwURLString = state.gwURL.toString()
   delete state.customGatewayUrl
   state.dnslinkPolicy = String(options.dnslinkPolicy) === 'false' ? false : options.dnslinkPolicy
+  // store info about 'blessed' release of Web UI
+  // which should work without setting CORS headers
+  state.webuiCid = 'QmXc9raDM1M5G5fpBnVyQ71vR4gbnskwnB9iMEzBuLgvoZ' // v2.3.3
+  state.webuiRootUrl = `${state.gwURLString}ipfs/${state.webuiCid}/`
   return state
 }
 

add-on/src/popup/browser-action/operations.js

@@ -16,7 +16,7 @@ module.exports = function operations ({
   onToggleRedirect
 }) {
   const activeQuickUpload = active && isIpfsOnline && isApiAvailable
-  const activeWebUI = active && isIpfsOnline // (js-ipfs >=0.34.0-rc.0 is ok) && ipfsNodeType === 'external'
+  const activeWebUI = active && isIpfsOnline && ipfsNodeType === 'external'
   const activeGatewaySwitch = active && ipfsNodeType === 'external'
 
   return html`

add-on/src/popup/browser-action/store.js

@@ -125,8 +125,7 @@ module.exports = (state, emitter) => {
 
   emitter.on('openWebUi', async () => {
     try {
-      // Open bundled version of WebUI
-      await browser.tabs.create({ url: '/webui/index.html' })
+      browser.tabs.create({ url: state.webuiRootUrl })
       window.close()
     } catch (error) {
       console.error(`Unable Open Web UI due to ${error}`)
@@ -236,6 +235,7 @@ module.exports = (state, emitter) => {
       state.isIpfsOnline = state.active && status.peerCount > -1
       state.gatewayVersion = state.active && status.gatewayVersion ? status.gatewayVersion : null
       state.ipfsApiUrl = state.active ? options.ipfsApiUrl : null
+      state.webuiRootUrl = status.webuiRootUrl
     } else {
       state.ipfsNodeType = 'external'
       state.swarmPeers = null

package.json

@@ -23,12 +23,12 @@
     "build:copy:ui-kit:ipfs-css:fonts": "shx mkdir -p add-on/ui-kit/fonts && shx cp node_modules/ipfs-css/fonts/* add-on/ui-kit/fonts",
     "build:copy:ui-kit:ipfs-css:icons": "shx mkdir -p add-on/ui-kit/icons && shx cp node_modules/ipfs-css/icons/* add-on/ui-kit/icons",
     "build:copy:ui-kit:tachyons": "shx mkdir -p add-on/ui-kit && shx cp node_modules/tachyons/css/tachyons.css add-on/ui-kit",
-    "build:webui": "cross-env CID=QmXc9raDM1M5G5fpBnVyQ71vR4gbnskwnB9iMEzBuLgvoZ npm run build:webui:with-cid",
-    "build:webui:with-cid": "cross-env-shell \"shx test -d add-on/webui/ || (npm run build:webui:dir && (npm run build:webui:fetch-ipfs || npm run build:webui:fetch-http) && npm run build:webui:minimize)\"",
-    "build:webui:dir": "shx mkdir -p add-on/webui",
-    "build:webui:fetch-ipfs": "cross-env-shell \"ipfs get $CID -o add-on/webui/\"",
-    "build:webui:fetch-http": "cross-env-shell \"node scripts/fetch-webui-from-gateway.js $CID add-on/webui/\"",
-    "build:webui:minimize": "shx rm -rf add-on/webui/static/js/*.map && shx rm -rf add-on/webui/static/css/*.map && shx rm -rf add-on/webui/manifest.json",
+    "DISABLED:issue679:build:webui": "cross-env CID=QmXc9raDM1M5G5fpBnVyQ71vR4gbnskwnB9iMEzBuLgvoZ npm run build:webui:with-cid",
+    "DISABLED:issue679:build:webui:with-cid": "cross-env-shell \"shx test -d add-on/webui/ || (npm run build:webui:dir && (npm run build:webui:fetch-ipfs || npm run build:webui:fetch-http) && npm run build:webui:minimize)\"",
+    "DISABLED:issue679:build:webui:dir": "shx mkdir -p add-on/webui",
+    "DISABLED:issue679:build:webui:fetch-ipfs": "cross-env-shell \"ipfs get $CID -o add-on/webui/\"",
+    "DISABLED:issue679:build:webui:fetch-http": "cross-env-shell \"node scripts/fetch-webui-from-gateway.js $CID add-on/webui/\"",
+    "DISABLED:issue679:build:webui:minimize": "shx rm -rf add-on/webui/static/js/*.map && shx rm -rf add-on/webui/static/css/*.map && shx rm -rf add-on/webui/manifest.json",
     "build:js": "run-s build:js:*",
     "build:js:webpack": "webpack -p",
     "build:minimize-dist": "shx rm -rf add-on/dist/lib add-on/dist/contentScripts/ add-on/dist/bundles/ipfsProxyContentScriptPayload.bundle.js",
@@ -127,6 +127,7 @@
     "piggybacker": "2.0.0",
     "pull-file-reader": "1.0.2",
     "tachyons": "4.11.1",
+    "uri-to-multiaddr": "3.0.1",
     "webextension-polyfill": "0.3.1"
   }
 }

scripts/fetch-webui-from-gateway.js

@@ -1,5 +1,6 @@
 /* This is a fallback script used when ipfs cli fails or is not available
  * More details: https://github.com/ipfs-shipyard/ipfs-webui/issues/843
+ * See also why this is not used: https://github.com/ipfs-shipyard/ipfs-companion/issues/679
  */
 const tar = require('tar')
 const request = require('request')
@@ -8,10 +9,6 @@ const progress = require('request-progress')
 const cid = process.argv[2]
 const destination = process.argv[3]
 
-// pick random preloader
-// const no = Math.round(Math.random()) // 0 or 1
-// const url = 'https://node' + no + '.preload.ipfs.io/api/v0/get?arg=' + cid + '&archive=true&compress=true'
-
 // use public gw
 const url = 'https://ipfs.io/api/v0/get?arg=' + cid + '&archive=true&compress=true'