Move ledger data:* favicons to external files

This moves data:* URLs in the ledger appState to external image files, which
reduces the size of my session-store-1 from ~25 MB to 5 MB.

Fix #11582

Test plan:
1. Copy session-store-1 from a Brave profile that has had payments enabled for a while to your brave-development appData directory.
2. Start the browser and go to about:preferences#payments. The synopsis table should appear fine.
3. Close the browser.
4. In your brave-development appData directory, you should now see a ledger-favicons subdirectory which contains a lot of image files. You should also notice that session-store-1 is now much smaller.
5. Repeat step 2.

app/renderer/components/preferences/payment/ledgerTable.js

@@ -36,6 +36,12 @@ class LedgerTable extends ImmutableComponent {
     this.props.onChangeSetting(settings.PAYMENTS_SITES_SHOW_LESS, value)
   }
 
+  onFaviconError (faviconURL, publisherKey) {
+    console.log('missing or corrupted favicon file', faviconURL)
+    // Set the publishers favicon to null so that it gets refetched
+    aboutActions.setLedgerFavicon(publisherKey, null)
+  }
+
   getFormattedTime (synopsis) {
     var d = synopsis.get('daysSpent')
     var h = synopsis.get('hoursSpent')
@@ -170,7 +176,7 @@ class LedgerTable extends ImmutableComponent {
           <a className={css(styles.siteData__anchor)} href={publisherURL} rel='noopener' target='_blank' tabIndex={-1}>
             {
               faviconURL
-                ? <img className={css(styles.siteData__anchor__icon_favicon)} src={faviconURL} alt={siteName} />
+                ? <img className={css(styles.siteData__anchor__icon_favicon)} src={faviconURL} alt='' onError={this.onFaviconError.bind(null, faviconURL, publisherKey)} />
                 : <span className={css(styles.siteData__anchor__icon_default)}><span className={globalStyles.appIcons.defaultIcon} /></span>
             }
             <span className={css(styles.siteData__anchor__url)} data-test-id='siteName'>{siteName}</span>

app/sessionStore.js

@@ -37,6 +37,7 @@ const {defaultSiteSettingsList} = require('../js/data/siteSettingsList')
 const filtering = require('./filtering')
 const autofill = require('./autofill')
 const {navigatableTypes} = require('../js/lib/appUrlUtil')
+const {isDataUrl, parseFaviconDataUrl} = require('../js/lib/urlUtil')
 const Channel = require('./channel')
 const BuildConfig = require('./buildConfig')
 const {isImmutable, isMap, makeImmutable, deleteImmutablePaths} = require('./common/state/immutableUtil')
@@ -55,8 +56,8 @@ const getTempStoragePath = (filename) => {
     : path.join(process.env.HOME, '.brave-test-session-store-' + filename + '-' + epochTimestamp)
 }
 
-const getStoragePath = () => {
-  return path.join(app.getPath('userData'), sessionStorageName)
+const getStoragePath = (filename = sessionStorageName) => {
+  return path.join(app.getPath('userData'), filename)
 }
 /**
  * Saves the specified immutable browser state to storage.
@@ -421,7 +422,7 @@ module.exports.cleanAppData = (immutableData, isShutdown) => {
     })
   }
 
-  // Leader cleanup
+  // Ledger cleanup
   if (immutableData.has('pageData')) {
     immutableData = immutableData.delete('pageData')
   }
@@ -442,6 +443,21 @@ module.exports.cleanAppData = (immutableData, isShutdown) => {
     })
   }
 
+  try {
+    // Prune data: favicons by moving them to external files
+    const basePath = getStoragePath('ledger-favicons')
+    if (immutableData.get('createdFaviconDirectory') !== true) {
+      const fs = require('fs')
+      if (!fs.existsSync(basePath)) {
+        fs.mkdirSync(basePath)
+      }
+      immutableData = immutableData.set('createdFaviconDirectory', true)
+    }
+    immutableData = cleanFavicons(basePath, immutableData)
+  } catch (e) {
+    console.error('cleanAppData: error cleaning up data: urls', e)
+  }
+
   return immutableData
 }
 
@@ -461,6 +477,49 @@ module.exports.cleanSessionDataOnShutdown = () => {
   }
 }
 
+const cleanFavicons = (basePath, immutableData) => {
+  const fs = require('fs')
+  const synopsisPaths = [
+    // TODO (nejc) - remove duplicate entries in synopsis and about/synopsis
+    ['ledger', 'synopsis', 'publishers'],
+    ['ledger', 'about', 'synopsis']
+  ]
+  // Map of favicon content to location on disk to avoid saving dupes
+  const savedFavicons = {}
+  synopsisPaths.forEach((synopsisPath) => {
+    if (immutableData.getIn(synopsisPath)) {
+      immutableData.getIn(synopsisPath).forEach((value, index) => {
+        // Fix #11582
+        if (value && value.get && isDataUrl(value.get('faviconURL', ''))) {
+          const parsed = parseFaviconDataUrl(value.get('faviconURL'))
+          if (!parsed) {
+            immutableData = immutableData.setIn(
+              synopsisPath.concat([index, 'faviconURL']), '')
+            return
+          }
+          let faviconPath = savedFavicons[parsed.data]
+          if (!faviconPath) {
+            faviconPath = path.join(basePath,
+              typeof index === 'number'
+                ? `${Date.now()}.${parsed.ext}`
+                : `${index.replace(/[^a-z0-9]/gi, '_')}.${parsed.ext}`
+            )
+            savedFavicons[parsed.data] = faviconPath
+            fs.writeFile(faviconPath, parsed.data, 'base64', (err) => {
+              if (err) {
+                console.error(`Error writing file: ${faviconPath} ${err}`)
+              }
+            })
+          }
+          immutableData = immutableData.setIn(
+            synopsisPath.concat([index, 'faviconURL']), `file://${faviconPath}`)
+        }
+      })
+    }
+  })
+  return immutableData
+}
+
 const safeGetVersion = (fieldName, getFieldVersion) => {
   const versionField = {
     name: fieldName,

docs/state.md

@@ -180,6 +180,10 @@ AppStore
     }
   },
   ledger: {
+    about: {
+      synopsis: Array.Object,
+      synopsisOptions: Object
+    },
     info: {
       addresses: {
         BAT: string, 
@@ -340,7 +344,7 @@ AppStore
       publishers: {
         [publisherId]: {
           duration: number,
-          faviconUrl: string,
+          faviconURL: string,
           options: {
             exclude: boolean,
             verified: boolean,
@@ -683,6 +687,7 @@ WindowStore
     }],
     top: number // the top position of the context menu
   },
+  createdFaviconDirectory: boolean, // whether the ledger-favicons directory has been created already in the appData directory
   frames: [{
     aboutDetails: object, // details for about pages
     activeShortcut: string, // set by the application store when the component should react to a shortcut

js/about/aboutActions.js

@@ -154,6 +154,19 @@ const aboutActions = {
     })
   },
 
+  /**
+   * Sets ledger publisher favicon property
+   * @param {string} publisherKey
+   * @param {string?} blob
+   */
+  setLedgerFavicon: function (publisherKey, blob) {
+    aboutActions.dispatchAction({
+      actionType: appConstants.APP_ON_FAVICON_RECEIVED,
+      publisherKey,
+      blob
+    })
+  },
+
   /**
    * Click through a certificate error.
    *

js/lib/urlutil.js

@@ -205,6 +205,33 @@ const UrlUtil = {
     return typeof url === 'string' && url.toLowerCase().startsWith('data:')
   },
 
+  /**
+   * Parses a favicon data URL
+   * @param {String} url The data URL
+   * @returns {{data: String, ext: String}?}
+   */
+  parseFaviconDataUrl: function (url) {
+    if (!UrlUtil.isDataUrl(url)) {
+      return null
+    }
+    const parsed = {}
+    url = url.slice(5) // slice off 'data:' prefix
+    const header = url.split(',')[0]
+    if (!header || !header.includes(';base64')) {
+      return null
+    }
+    const mimeType = header.split(';')[0]
+    if (!mimeType.startsWith('image/')) {
+      return null
+    }
+    parsed.ext = mimeType.split('/')[1]
+    parsed.data = url.split(',')[1]
+    if (parsed.data && parsed.ext) {
+      return parsed
+    }
+    return null
+  },
+
   /**
    * Checks if a url is a phishable url.
    * @param {String} input The input url.

test/unit/lib/urlutilTest.js

@@ -417,4 +417,59 @@ describe('urlutil', function () {
       assert.equal(result, 'https://brave.com')
     })
   })
+
+  describe('parseFaviconDataUrl', function () {
+    it('null scenario', function () {
+      const result = urlUtil.parseFaviconDataUrl(null)
+      assert.equal(result, null)
+    })
+    it('empty string', function () {
+      const result = urlUtil.parseFaviconDataUrl('')
+      assert.equal(result, null)
+    })
+    it('regular URL', function () {
+      const result = urlUtil.parseFaviconDataUrl('http://example.com')
+      assert.equal(result, null)
+    })
+    it('non-image data URL', function () {
+      const result = urlUtil.parseFaviconDataUrl('data:text/plain;charset=UTF-8;page=21,the%20data:1234,5678')
+      assert.equal(result, null)
+    })
+    it('non-base64 data URL', function () {
+      const result = urlUtil.parseFaviconDataUrl('data:image/jpg,foo')
+      assert.equal(result, null)
+    })
+    it('no-extension data URL', function () {
+      const result = urlUtil.parseFaviconDataUrl('data:image/;base64,foo')
+      assert.equal(result, null)
+    })
+    it('valid jpg', function () {
+      const jpg = 'data:image/jpeg;base64,' +
+        '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDADIiJSwlHzIsKSw4NTI7S31RS0VFS5ltc1p9tZ++u7Kf' +
+        'r6zI4f/zyNT/16yv+v/9////////wfD/////////////2wBDATU4OEtCS5NRUZP/zq/O////////' +
+        '////////////////////////////////////////////////////////////wAARCAAYAEADAREA' +
+        '//AhEBAxEB/8QAGQAAAgMBAAAAAAAAAAAAAAAAAQMAAgQF/8QAJRABAAIBBAEEAgMAAAAAAAAAAQIR' +
+        '//AAMSITEEEyJBgTORUWFx/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAA' +
+        '//AAD/2gAMAwEAAhEDEQA/AOgM52xQDrjvAV5Xv0vfKUALlTQfeBm0HThMNHXkL0Lw/swN5qgA8yT4' +
+        '//MCS1OEOJV8mBz9Z05yfW8iSx7p4j+jA1aD6Wj7ZMzstsfvAas4UyRHvjrAkC9KhpLMClQntlqFc2' +
+        '//X1gUj4viwVObKrddH9YDoHvuujAEuNV+bLwFS8XxdSr+Cq3Vf+4F5RgQl6ZR2p1eAzU/HX80YBYy' +
+        '//JLCuexwJCO2O1bwCRidAfWBSctswbI12GAJT3yiwFR7+MBjGK2g/WAJR3FdF84E2rK5VR0YH/9k='
+      const expected = '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDADIiJSwlHzIsKSw4NTI7S31RS0VFS5ltc1p9tZ++u7Kf' +
+        'r6zI4f/zyNT/16yv+v/9////////wfD/////////////2wBDATU4OEtCS5NRUZP/zq/O////////' +
+        '////////////////////////////////////////////////////////////wAARCAAYAEADAREA' +
+        '//AhEBAxEB/8QAGQAAAgMBAAAAAAAAAAAAAAAAAQMAAgQF/8QAJRABAAIBBAEEAgMAAAAAAAAAAQIR' +
+        '//AAMSITEEEyJBgTORUWFx/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAA' +
+        '//AAD/2gAMAwEAAhEDEQA/AOgM52xQDrjvAV5Xv0vfKUALlTQfeBm0HThMNHXkL0Lw/swN5qgA8yT4' +
+        '//MCS1OEOJV8mBz9Z05yfW8iSx7p4j+jA1aD6Wj7ZMzstsfvAas4UyRHvjrAkC9KhpLMClQntlqFc2' +
+        '//X1gUj4viwVObKrddH9YDoHvuujAEuNV+bLwFS8XxdSr+Cq3Vf+4F5RgQl6ZR2p1eAzU/HX80YBYy' +
+        '//JLCuexwJCO2O1bwCRidAfWBSctswbI12GAJT3yiwFR7+MBjGK2g/WAJR3FdF84E2rK5VR0YH/9k='
+      const result = urlUtil.parseFaviconDataUrl(jpg)
+      assert.deepEqual(result, {data: expected, ext: 'jpeg'})
+    })
+    it('valid png', function () {
+      const png = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU//5ErkJggg=='
+      const result = urlUtil.parseFaviconDataUrl(png)
+      assert.deepEqual(result, {data: 'iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU//5ErkJggg==', ext: 'png'})
+    })
+  })
 })