Make tests intermittent resistant

Fix #9799

Auditors: @aekeus

test/intermittents.txt

@@ -0,0 +1,33 @@
+#Ledger
+Advanced payment panel tests can backup wallet to file
+Advanced payment panel tests can recover wallet from file
+Advanced payment panel tests shows an error popover if one recovery key is missing
+Advanced payment panel tests shows an error popover if a recovery key is not a UUID
+Advanced payment panel tests shows an error popover if both recovery keys are missing
+Advanced payment panel tests shows an error popover if the file is empty
+Regular payment panel tests can setup payments “before each” hook for “shows welcome page”
+Ledger table 2 publishers check pinned sites amount, when you have 0 eligible unpinned sites
+
+#Navigationbar
+Bravery Panel Adblock stats without iframe tests detects https upgrades in private tab
+Bravery Panel Adblock stats without iframe tests block device enumeration
+noscript info can allow scripts on a file:// URL without allowing all scripts
+noscript info can selectively allow scripts once
+Clear Browsing Panel with saved state values saves the history switch state
+
+#Sync
+Syncing bookmarks from an existing profile “before all” hook
+Syncing and clearing data prevents it from syncing history
+
+#Tabs
+tab tests webview previews when tab is hovered does not show tab previews when setting is off
+tab tests tab transfer can detach into new windows
+
+#URLbar
+navigationBar tests navigation User input remains cleared when onChange is fired but not onKeyUp
+navigationBar tests lockIcon Temporarily allow/deny running insecure content
+navigationBar tests submit with no url input value shows search icon when input is empty
+navigationBar tests submit with about page url input values shows the list icon in URL bar for other about pages
+urlBar tests autocomplete “before each” hook for “autocompletes with protocol”
+urlBar tests loading same URL as current page with changed input “before all” hook
+urlBar tests keeps url text separate from suffix text changes only the selection

tools/test.js

@@ -2,23 +2,81 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-var execute = require('./lib/execute')
+const fs = require('fs')
+const spawn = require('child_process').spawn
 
 const TEST_DIR = process.env.TEST_DIR
 
-var cmd = []
+const options = {
+  env: process.env,
+  shell: true
+}
+
+const intermittents = fs.readFileSync('./test/intermittents.txt')
+  .toString()
+  .split('\n')
+  .filter((intermittent) => !intermittent.match(/^\s*$/) && !intermittent.match(/s*#/))
 
-if (TEST_DIR === 'lint') {
-  cmd.push('standard')
-} else {
-  cmd.push(`mocha "test/${TEST_DIR}/**/*Test.js"`)
+let testCommand = 'standard'
+if (TEST_DIR !== 'lint') {
+  testCommand = `mocha "test/${TEST_DIR}/**/*Test.js"`
 }
+const child = spawn(testCommand, options)
+
+let output = ''
+child.stdout.on('data', function (data) {
+  const newData = data.toString()
+  output += newData
+  process.stdout.write(newData);
+})
+
+const matchFailing = (output) =>
+  output.match(/\s+(\d+) failing\n/)
+
+child.on('close', function (code) {
+  if (code === 0) {
+    console.log('Result code is 0, no aciton needed.')
+    process.exit(0)
+    return
+  }
+
+  // There;'s no such thing as an intermittent lint failures
+  if (TEST_DIR === 'lint') {
+    process.exit(1)
+    return
+  }
 
-execute(cmd, process.env, (err) => {
-  if (err) {
-    console.error('failed', err)
+  console.log('Result code is not 0, let\'s check for intermittents...')
+  const matchResult = matchFailing(output)
+  if (!matchResult) {
+    console.error('Could not find line which gives the number of failures.')
     process.exit(1)
     return
   }
-  console.log('done')
+  const expectedFailing = Number(matchResult[1])
+  output = output.substring(matchResult.index)
+
+  // Check if there were failures
+  const results = output.split('\n')
+  const failures = []
+  results.forEach((result) => {
+    const failingLineMatch = result.match(/\d+\) (.*):$/)
+    if (failingLineMatch) {
+      const match = failingLineMatch[1]
+      failures.push(match)
+    }
+  })
+  if (failures.length !== expectedFailing) {
+    console.error('Could not extract the correct number of failing tests', failures.length, 'vs', expectedFailing)
+    process.exit(1)
+    return
+  }
+
+  const unknownFailures = failures.filter((failure) => !intermittents.includes(failure))
+  if (unknownFailures.length === 0) {
+    console.log('Only known intermittent failures were hit. Returning success!')
+    process.exit(0)
+  }
+  console.log('Unknown failures exist:', unknownFailures)
+  process.exit(1)
 })