From b1f26e2955211003bdf1fd3f445a5c39c1235a81 Mon Sep 17 00:00:00 2001 From: Christopher Hiller Date: Wed, 4 Nov 2020 11:39:56 -0800 Subject: [PATCH] handful of improvements to integration tests - refactor assertion types - add `test/integration/README.md` - add support for `MOCHA_TEST_KEEP_TEMP_DIRS` env var --- test/assertions.js | 26 ++++++++++-------------- test/integration/README.md | 40 +++++++++++++++++++++++++++++++++++++ test/integration/helpers.js | 25 +++++++++++------------ 3 files changed, 63 insertions(+), 28 deletions(-) create mode 100644 test/integration/README.md diff --git a/test/assertions.js b/test/assertions.js index 8dd682d9e8..f5f7dd20ba 100644 --- a/test/assertions.js +++ b/test/assertions.js @@ -13,41 +13,37 @@ module.exports = { base: 'object', identify(v) { return ( - v !== null && - typeof v === 'object' && + this.baseType.identify(v) && typeof v.output === 'string' && 'code' in v && // may be null - Array.isArray(v.args) + Array.isArray(v.args) && + typeof v.command === 'string' ); } }) .addType({ name: 'JSONResult', - base: 'object', + base: 'RawResult', identify(v) { return ( - v !== null && - typeof v === 'object' && - v.stats !== null && + this.baseType.identify(v) && typeof v.stats === 'object' && Array.isArray(v.failures) && - typeof v.code === 'number' && - typeof v.command === 'string' + Array.isArray(v.passes) && + Array.isArray(v.tests) && + Array.isArray(v.pending) ); } }) .addType({ name: 'SummarizedResult', - base: 'object', + base: 'RawResult', identify(v) { return ( - v !== null && - typeof v === 'object' && + this.baseType.identify(v) && typeof v.passing === 'number' && typeof v.failing === 'number' && - typeof v.pending === 'number' && - typeof v.output === 'string' && - typeof v.code === 'number' + typeof v.pending === 'number' ); } }) diff --git a/test/integration/README.md b/test/integration/README.md new file mode 100644 index 0000000000..5b2e84d750 --- /dev/null +++ b/test/integration/README.md @@ -0,0 +1,40 @@ +# Mocha Integration Tests + +The tests in this directory are integration or end-to-end tests. Most of them spawn a `mocha` process and inspect the result of `STDOUT` and/or `STDERR`. + +## Directories + +- `cli`: Related to general CLI behavior; not necessarily command-line-flag specific +- `fixtures`: Test file fixtures intended to be run via these tests. Usually have `.fixture.js` extension +- `plugins`: Tests related to plugins (e.g., root hook plugins, global fixtures, etc.) +- `options`: Tests for specific command-line flags + +## Helpers + +The `helpers.js` module contains many functions to handle the common cases of spawning a Mocha process and other utilities. The important ones: + +- `runMocha` / `runMochaAsync`: spawns Mocha to run a fixture with the default reporter. Returns a parsed `SummarizedResult` object containing information parsed from the reporter's epilogue +- `runMochaJSON` / `runMochaJSONAsync`: spawns Mocha to run a fixture with a `json` reporter and parses the output; good for assertions about specific numbers and types of test results. Returns a `JSONResult` object +- `invokeMocha` / `invokeMochaAsync`: spawns Mocha with the default reporter but does not parse its output; good for testing errors of the non-test-failure variety. Does not expect a fixture file path, but one can manually be provided. Preferred to test Mocha's output to `STDERR`. Returns a `RawSummarizedResultObject` with the raw output and exit code, etc. +- `resolveFixurePath`: a handy way to get the path to a fixture file. Required when using `invokeMocha*` +- `runMochaWatch*`: similar to `runMocha*`, but runs Mocha in "watch" mode. Accepts a function which should trigger a rerun; the function should touch a file or perform some other filesystem operation. Forks instead of spawns on Windows +- `invokeNode`: spawns `node` instead of `mocha`; good for testing programmatic usage of Mocha by running a script which does this + +### Return Types + +- `RawResult`: an object containing props `args`, `code`, `output` and `command` +- `SummarizedResult`: a `RawResult` + props `passing`, `failing` and `pending` +- `JSONResult`: a `RawResult` + parsed output of `json` reporter + +### Default Arguments + +By default, all of these helpers run with the following options: + +- `--no-color`: it's easier to make assertions about output w/o having to deal w/ ANSI escape codes +- `--no-bail`: overrides a configuration file w/ `bail: true`; providing `--bail` to the arguments list will supress this (useful when testing `--bail`!) +- `--no-parallel`: overrides a configuration file w/ `parallel: true`; providing `--parallel` to the arguments list will suppress this + +## Environment Variables Which Do Stuff + +- `DEBUG=mocha:test*`: will show debug output from tests & helpers, if any +- `MOCHA_TEST_KEEP_TEMP_DIRS=1`: does not automatically remove any temporary directories and files created by the `createTempDir` helper. Use to manually debug problems when running fixtures in temp directories diff --git a/test/integration/helpers.js b/test/integration/helpers.js index ccb20c9b11..5744bb58c1 100644 --- a/test/integration/helpers.js +++ b/test/integration/helpers.js @@ -6,7 +6,7 @@ const fs = require('fs-extra'); const {format} = require('util'); const path = require('path'); const Base = require('../../lib/reporters/base'); -const debug = require('debug')('mocha:tests:integration:helpers'); +const debug = require('debug')('mocha:test:integration:helpers'); const touch = require('touch'); /** @@ -133,11 +133,11 @@ function runMochaJSON(fixturePath, args, done, opts) { * @returns {Promise} */ function runMochaAsync(fixturePath, args, opts) { - return new Promise(function(resolve, reject) { + return new Promise((resolve, reject) => { runMocha( fixturePath, args, - function(err, result) { + (err, result) => { if (err) { return reject(err); } @@ -156,11 +156,11 @@ function runMochaAsync(fixturePath, args, opts) { * @returns {Promise} */ function runMochaJSONAsync(fixturePath, args, opts) { - return new Promise(function(resolve, reject) { + return new Promise((resolve, reject) => { runMochaJSON( fixturePath, args, - function(err, result) { + (err, result) => { if (err) { return reject(err); } @@ -178,9 +178,8 @@ function runMochaJSONAsync(fixturePath, args, opts) { * @returns {JSONResult} */ function toJSONResult(result) { - const {code, command, output} = result; try { - return {...JSON.parse(output), code, command}; + return {...JSON.parse(result.output), ...result}; } catch (err) { throw new Error( `Couldn't parse JSON: ${err.message}\n\nOriginal result output: ${result.output}` @@ -383,11 +382,9 @@ function resolveFixturePath(fixture) { * @returns {Summary} */ function getSummary(res) { - return ['passing', 'pending', 'failing'].reduce(function(summary, type) { - var pattern, match; - - pattern = new RegExp(' (\\d+) ' + type + '\\s'); - match = pattern.exec(res.output); + return ['passing', 'pending', 'failing'].reduce((summary, type) => { + const pattern = new RegExp(` (\\d+) ${type}\\s`); + const match = pattern.exec(res.output); summary[type] = match ? parseInt(match, 10) : 0; return summary; @@ -524,7 +521,9 @@ const createTempDir = async () => { return { dirpath, removeTempDir: async () => { - return fs.remove(dirpath); + if (!process.env.MOCHA_TEST_KEEP_TEMP_DIRS) { + return fs.remove(dirpath); + } } }; };