diff --git a/app/ledger.js b/app/ledger.js index 65d1a7096f8..455c562f22d 100644 --- a/app/ledger.js +++ b/app/ledger.js @@ -33,13 +33,14 @@ const ipc = electron.ipcMain const session = electron.session const acorn = require('acorn') +const levelup = require('level') +const moment = require('moment') const qr = require('qr-image') const querystring = require('querystring') const random = require('random-lib') const tldjs = require('tldjs') const underscore = require('underscore') const uuid = require('node-uuid') -const moment = require('moment') const appActions = require('../js/actions/appActions') const appConfig = require('../js/constants/appConfig') @@ -86,6 +87,12 @@ const clientOptions = { debugP: process.env.LEDGER_DEBUG, var doneTimer +var v2RulesetDB +const v2RulesetPath = 'ledger-rulesV2.leveldb' + +var v2PublishersDB +const v2PublishersPath = 'ledger-publishersV2.leveldb' + /* * publisher globals */ @@ -190,6 +197,7 @@ const doAction = (action) => { if (publisherInfo._internal.verboseP) console.log('\nupdating ' + publisher + ' stickyP=' + action.value) synopsis.publishers[publisher].options.stickyP = action.value updatePublisherInfo() + verifiedP(publisher) } break @@ -232,6 +240,14 @@ var quit = () => { visit('NOOP', underscore.now(), null) clearInterval(doneTimer) doneWriter() + if (v2RulesetDB) { + v2RulesetDB.close() + v2RulesetDB = null + } + if (v2PublishersDB) { + v2PublishersDB.close() + v2PublishersDB = null + } } var boot = () => { @@ -373,7 +389,7 @@ var recoverKeys = (appState, action) => { return appState } - client.recoverWallet(firstRecoveryKey, secondRecoveryKey, (err, body) => { + client.recoverWallet(firstRecoveryKey, secondRecoveryKey, (err, result) => { let existingLedgerError = ledgerInfo.error if (logError(err, 'recoveryWallet')) { @@ -384,6 +400,8 @@ var recoverKeys = (appState, action) => { appActions.updateLedgerInfo(underscore.omit(ledgerInfo, [ '_internal' ])) setImmediate(() => appActions.ledgerRecoveryFailed()) } else { + callback(err, result) + appActions.updateLedgerInfo(underscore.omit(ledgerInfo, [ '_internal' ])) if (balanceTimeoutId) clearTimeout(balanceTimeoutId) getBalance() @@ -532,6 +550,7 @@ underscore.keys(fileTypes).forEach((fileType) => { signatureMax = Math.ceil(signatureMax * 1.5) eventStore.addChangeListener(() => { + var initP const eventState = eventStore.getState().toJS() var view = eventState.page_view var info = eventState.page_info @@ -548,7 +567,7 @@ eventStore.addChangeListener(() => { if (!page.publisher) { try { - publisher = ledgerPublisher.getPublisher(location) + publisher = ledgerPublisher.getPublisher(location, publisherInfo._internal.ruleset.raw) if (publisher) { siteSetting = appStore.getState().get('siteSettings').get(`https?://${publisher}`) if ((siteSetting) && (siteSetting.get('ledgerPaymentsShown') === false)) publisher = null @@ -563,10 +582,14 @@ eventStore.addChangeListener(() => { publisher = page.publisher pattern = `https?://${publisher}` - if ((!synopsis.publishers[publisher]) && (!getSetting(settings.AUTO_SUGGEST_SITES))) { - appActions.changeSiteSetting(pattern, 'ledgerPayments', false) - } + initP = !synopsis.publishers[publisher] synopsis.initPublisher(publisher) + if ((initP) && (getSetting(settings.AUTO_SUGGEST_SITES))) { + excludeP(publisher, (unused, exclude) => { + appActions.changeSiteSetting(pattern, 'ledgerPayments', !exclude) + updatePublisherInfo() + }) + } entry = synopsis.publishers[publisher] if ((page.protocol) && (!entry.protocol)) entry.protocol = page.protocol @@ -647,12 +670,14 @@ eventStore.addChangeListener(() => { */ var initialize = (paymentsEnabled) => { + var ruleset + + if (!v2RulesetDB) v2RulesetDB = levelup(pathName(v2RulesetPath)) + if (!v2PublishersDB) v2PublishersDB = levelup(pathName(v2PublishersPath)) enable(paymentsEnabled) // Check if relevant browser notifications should be shown every 15 minutes - if (notificationTimeout) { - clearInterval(notificationTimeout) - } + if (notificationTimeout) clearInterval(notificationTimeout) notificationTimeout = setInterval(showNotifications, 15 * msecs.minute) if (!paymentsEnabled) { @@ -662,7 +687,9 @@ var initialize = (paymentsEnabled) => { if (client) return if (!ledgerPublisher) ledgerPublisher = require('ledger-publisher') - cacheRuleSet(ledgerPublisher.rules) + ruleset = [] + ledgerPublisher.ruleset.forEach(rule => { if (rule.consequent) ruleset.push(rule) }) + cacheRuleSet(ruleset) fs.access(pathName(statePath), fs.FF_OK, (err) => { if (!err) { @@ -749,7 +776,7 @@ var enable = (paymentsEnabled) => { synopsis = new (ledgerPublisher.Synopsis)() fs.readFile(pathName(synopsisPath), (err, data) => { var initSynopsis = () => { - var updateP, value + var value var siteSettings = appStore.getState().get('siteSettings') // cf., the `Synopsis` constructor, https://github.com/brave/ledger-publisher/blob/master/index.js#L167 @@ -787,14 +814,15 @@ var enable = (paymentsEnabled) => { underscore.keys(synopsis.publishers).forEach((publisher) => { var siteSetting + excludeP(publisher) + verifiedP(publisher) if (typeof synopsis.publishers[publisher].options.stickyP !== 'undefined') return - updateP = true siteSetting = siteSettings.get(`https?://${publisher}`) synopsis.publishers[publisher].options.stickyP = siteSetting && siteSetting.get('ledgerPayments') }) - if (updateP) updatePublisherInfo() + updatePublisherInfo() } if (publisherInfo._internal.verboseP) console.log('\nstarting up ledger publisher integration') @@ -901,8 +929,9 @@ var synopsisNormalizer = () => { results = [] underscore.keys(synopsis.publishers).forEach((publisher) => { - if ((getSetting(settings.AUTO_SUGGEST_SITES)) && (!synopsis.publishers[publisher].options.stickyP)) { - if ((synopsis.publishers[publisher].scores[scorekeeper] <= 0) || + if (!synopsis.publishers[publisher].options.stickyP) { + if ((synopsis.publishers[publisher].options.exclude === true) || + (synopsis.publishers[publisher].scores[scorekeeper] <= 0) || (synopsis.options.minPublisherDuration > synopsis.publishers[publisher].duration) || (synopsis.options.minPublisherVisits > synopsis.publishers[publisher].visits)) return } @@ -923,7 +952,7 @@ var synopsisNormalizer = () => { data[i] = { rank: i + 1, - verified: (ledgerInfo._internal.verifiedPublishers || []).indexOf(results[i].publisher) !== -1, + verified: results[i].options.verified || false, site: results[i].publisher, views: results[i].visits, duration: duration, @@ -1023,6 +1052,7 @@ var visit = (location, timestamp, tabId) => { } synopsis.addPublisher(publisher, { duration: duration, revisitP: revisitP }) updatePublisherInfo() + verifiedP(publisher) } setLocation() @@ -1094,6 +1124,100 @@ var cacheRuleSet = (ruleset) => { } } +var excludeP = (publisher, callback) => { + var doneP + + var done = (err, result) => { + doneP = true + if ((!err) && (typeof result !== 'undefined') && (!!synopsis.publishers[publisher]) && + (synopsis.publishers[publisher].options.exclude !== result)) { + synopsis.publishers[publisher].options.exclude = result + updatePublisherInfo() + } + + if (callback) callback(err, result) + } + + inspectP(v2RulesetDB, v2RulesetPath, publisher, 'exclude', 'domain:' + publisher, (err, result) => { + var props + + if (!err) return done(err, result.exclude) + + props = ledgerPublisher.getPublisherProps('https://' + publisher) + if (!props) return done() + + v2RulesetDB.createReadStream({ lt: 'domain:' }).on('data', (data) => { + var regexp, result, sldP, tldP + + if (doneP) return + + sldP = data.key.indexOf('SLD:') !== 0 + tldP = data.key.indexOf('TLD:') !== 0 + if ((!tldP) && (!sldP)) return + + if (underscore.intersection(data.key.split(''), + [ '^', '$', '*', '+', '?', '[', '(', '{', '|' ]).length === 0) { + if ((data.key !== ('TLD:' + props.TLD)) && (data.key !== ('SLD:' + props.SLD.split('.')[0]))) return + } else { + try { + regexp = new RegExp(data.key.substr(4)) + if (!regexp.test(props[tldP ? 'TLD' : 'SLD'])) return + } catch (ex) { + console.log(v2RulesetPath + ' stream invalid regexp ' + data.key + ': ' + ex.toString()) + } + } + + try { + result = JSON.parse(data.value) + } catch (ex) { + console.log(v2RulesetPath + ' stream invalid JSON ' + data.entry + ': ' + data.value) + } + + done(null, result.exclude) + }).on('error', (err) => { + console.log(v2RulesetPath + ' stream error: ' + JSON.stringify(err, null, 2)) + }).on('close', () => { + }).on('end', () => { + if (!doneP) done(null, false) + }) + }) +} + +var verifiedP = (publisher, callback) => { + inspectP(v2PublishersDB, v2PublishersPath, publisher, 'verified', null, callback) +} + +var inspectP = (db, path, publisher, property, key, callback) => { + var done = (err, result) => { + if ((!err) && (typeof result !== 'undefined') && (!!synopsis.publishers[publisher]) && + (synopsis.publishers[publisher].options[property] !== result[property])) { + synopsis.publishers[publisher].options[property] = result[property] + updatePublisherInfo() + } + + if (callback) callback(err, result) + } + + if (!key) key = publisher + db.get(key, (err, value) => { + var result + + if (err) { + if (!err.notFound) console.log(path + ' get ' + key + ' error: ' + JSON.stringify(err, null, 2)) + return done(err) + } + + try { + result = JSON.parse(value) + } catch (ex) { + console.log(v2RulesetPath + ' stream invalid JSON ' + key + ': ' + value) + result = {} + } + + done(null, result) + }) +} + /* * update ledger information */ @@ -1167,7 +1291,6 @@ var ledgerInfo = { exchangeInfo: undefined, _internal: { - verifiedPublishers: [], exchangeExpiry: 0, exchanges: {}, geoipExpiry: 0 @@ -1236,7 +1359,7 @@ var updateLedgerInfo = () => { var logs = [] var callback = (err, result, delayTime) => { - var i, then + var i, results, then var entries = client && client.report() var now = underscore.now() @@ -1272,6 +1395,44 @@ var callback = (err, result, delayTime) => { getPaymentInfo() } cacheRuleSet(result.ruleset) + if (result.rulesetV2) { + results = result.rulesetV2 + delete result.rulesetV2 + + entries = [] + results.forEach((entry) => { + entries.push({ type: 'put', + key: entry.facet + ':' + entry.publisher, + value: JSON.stringify(underscore.omit(entry, [ 'facet', 'publisher' ])) + }) + }) + + v2RulesetDB.batch(entries, (err) => { + if (err) return console.log(v2RulesetPath + ' error: ' + JSON.stringify(err, null, 2)) + + underscore.keys(synopsis.publishers).forEach((publisher) => { excludeP(publisher) }) + }) + } + if (result.publishersV2) { + results = result.publishersV2 + delete result.publishersV2 + + entries = [] + results.forEach((entry) => { + entries.push({ type: 'put', + key: entry.publisher, + value: JSON.stringify(underscore.omit(entry, [ 'publisher' ])) + }) + if ((synopsis.publishers[entry.publisher]) && + (synopsis.publishers[entry.publisher].options.verified !== entry.verified)) { + synopsis.publishers[entry.publisher].options.verified = entry.verified + updatePublisherInfo() + } + }) + v2PublishersDB.batch(entries, (err) => { + if (err) return console.log(v2PublishersPath + ' error: ' + JSON.stringify(err, null, 2)) + }) + } atomicWriter(pathName(statePath), result, { flushP: true }, () => {}) run(delayTime) @@ -1364,8 +1525,7 @@ var run = (delayTime) => { var siteSetting = siteSettings.get(`https?://${winner}`) if ((siteSetting) && - (siteSetting.get('ledgerPayments') === false || - siteSetting.get('ledgerPaymentsShown') === false)) return + (siteSetting.get('ledgerPayments') === false || siteSetting.get('ledgerPaymentsShown') === false)) return result = client.vote(winner) if (result) state = result @@ -1428,11 +1588,6 @@ var getStateInfo = (state) => { cacheReturnValue() } - if (!underscore.isEqual(ledgerInfo._internal.verifiedPublishers, state.verifiedPublishers)) { - ledgerInfo._internal.verifiedPublishers = state.verifiedPublishers - updatePublisherInfo() - } - ledgerInfo.transactions = [] if (!state.transactions) return updateLedgerInfo() diff --git a/package.json b/package.json index a2272423117..21c83f997c9 100644 --- a/package.json +++ b/package.json @@ -96,9 +96,9 @@ "keytar": "^3.0.0", "l20n": "^3.5.1", "ledger-balance": "^0.8.64", - "ledger-client": "^0.8.106", + "ledger-client": "^0.8.129", "ledger-geoip": "^0.8.75", - "ledger-publisher": "^0.8.108", + "ledger-publisher": "^0.8.124", "lru-cache": "^1.0.0", "moment": "^2.15.1", "niceware": "^1.0.4",